diff options
author | Freddie Liardet <frederick.liardet@arm.com> | 2021-08-03 15:57:32 +0100 |
---|---|---|
committer | Georgios Pinitas <georgios.pinitas@arm.com> | 2021-09-07 11:13:44 +0000 |
commit | d216f570750b8ccde3754c4aef53fc20a90cb32d (patch) | |
tree | 83a88d3d4391c6a8ca5dabc73c763e6f0878c595 /src/core/NEON/kernels/arm_conv/depthwise/kernels | |
parent | 4e53c5ab47a713ab0ce53d076e2e4cf274fec312 (diff) | |
download | ComputeLibrary-d216f570750b8ccde3754c4aef53fc20a90cb32d.tar.gz |
Update cpu depthwise kernels
Resolves: COMPMID-4688
Signed-off-by: Freddie Liardet <frederick.liardet@arm.com>
Change-Id: I9e22f967f5b7ccaebff2fc49f0253f621d62d820
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6030
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez Tello <pablo.tello@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Diffstat (limited to 'src/core/NEON/kernels/arm_conv/depthwise/kernels')
67 files changed, 4399 insertions, 4061 deletions
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index bb43d57018..be50d1c535 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst +class a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index 143070cc7e..a85e44360e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -721,4 +721,4 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 35a9714944..a0a44997d1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -696,4 +696,4 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 90db8703b5..39fa7f62fd 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst +class a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index c2d67661f6..9b4b3ee50d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1156,4 +1156,4 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index fc988913be..e0abca91a2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1289,4 +1289,4 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index df5328724d..1e0d922be4 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst +class a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index c403576cf1..b5bee7ae7c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1734,4 +1734,4 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 2e877285ae..e493104c03 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -2006,4 +2006,4 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index ca367cc1af..d89ae0cd6d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst +class a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 9b2d77fa53..a5df51c4f9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -893,4 +893,4 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 4857077b05..61c58186f5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -896,4 +896,4 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 53d2a3a8e1..6b5f91fa64 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst +class a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index 027fd7f4c7..b08059db0a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1385,4 +1385,4 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 43eac0e6bc..5b086ec1ff 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1426,4 +1426,4 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 88f20bb125..a888eb5776 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst +class a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index fae208fbab..73c1e07420 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -522,3 +524,5 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 2f93a68c23..42931fba17 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -509,3 +511,5 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 6a882ec52f..01bb06a561 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst +class a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 401528aa59..6d185e7274 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -823,3 +825,5 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 39ec001ae1..aa7d35e3e1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -901,3 +903,5 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 84bac12429..17084b57d5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst +class a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 616fd0d0e7..6faacf144a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -1227,3 +1229,5 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 51a5679bff..aeaf1049f1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -1393,3 +1395,5 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 8eb560562b..f23862b699 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst +class a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 4466ec1974..65e487ddbb 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -610,3 +612,5 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index a5153019e7..4b24862eb2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -625,3 +627,5 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 314fe766de..e4bfbe6783 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst +class a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index b74a3c9b7c..8b030ecc8b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -986,3 +988,5 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index adc5170ef7..4754a6f6f1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include <cstddef> #include <cstdint> +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -1039,3 +1041,5 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 14e113b776..b62ebb1687 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include <cstddef> #include <cstdint> diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index ccdde41973..8d22836e5b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include <cstddef> #include <cstdint> diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 699cc6c80c..b42f29ab0d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include <cstddef> #include <cstdint> diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index b8e59306d5..1cfea9dcbd 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst +class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index a4c1a40100..955a02de57 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,149 +113,141 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "add x27, x10, x22, LSL #1\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x26, x27, x22, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x14]\n" "add x25, x26, x22, LSL #1\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" "add x24, x11, x11\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" "add x23, x24, x11\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" "mul x19, x17, x20\n" // offset = tile_i * ld_output_row - "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" "madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col - "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" "mul x19, x19, x15\n" // offset *= output_tile_size - "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" "add x28, x28, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1) - "ld1h { z10.h }, p2/Z, [x10]\n" // Load input point (0, 0) + "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" "add x22, x28, x20, LSL #1\n" - "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3) + "whilelt p2.h, XZR, %x[n_channels]\n" + "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x10]\n" "addvl x14, x14, #16\n" - "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2) + "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n" "cmp x12, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" "addvl x14, x14, #-6\n" - "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1) + "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z4.h, z9.h\n" - "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" "whilelt p1.h, x12, %x[n_channels]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" "inch x21\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0) + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x25]\n" "inch x13\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2) + "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" "inch x12\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" + "ld1h { z16.h }, p3/Z, [x14]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z31.h, p3/M, z5.h, z12.h\n" "fmla z30.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1) + "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2) + "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" "addvl x10, x10, #1\n" "fmla z28.h, p3/M, z3.h, z13.h\n" "fmla z31.h, p3/M, z7.h, z13.h\n" "fmla z30.h, p3/M, z6.h, z13.h\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" "addvl x27, x27, #1\n" "fmla z29.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0) - "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z9.h }, p2/Z, [x26]\n" + "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" "fmla z28.h, p3/M, z2.h, z12.h\n" - "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" "fmla z31.h, p3/M, z8.h, z10.h\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" "addvl x26, x26, #1\n" "fmla z29.h, p3/M, z3.h, z9.h\n" - "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1) + "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2) + "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" "whilelt p2.h, x13, %x[n_channels]\n" "fmla z29.h, p3/M, z7.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" "addvl x25, x25, #1\n" "fmla z31.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1) + "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n" "cmp x12, %x[n_channels]\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x10]\n" // Load input point (0, 0) + "ld1h { z10.h }, p1/Z, [x10]\n" "fmla z28.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3) - "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n" + "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" "addvl x14, x14, #16\n" "fmax z31.h, p3/M, z31.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2) + "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" "addvl x14, x14, #-6\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0) - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x28]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z16.d\n" - "addvl x28, x28, #1\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "mov z29.d, z16.d\n" + "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" + "addvl x28, x28, #1\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1) - "mov z28.d, z16.d\n" + "st1h { z29.h }, p0, [x22]\n" + "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z4.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" "ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x17, #0x1\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0) + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x25]\n" "add x16, x16, #0x1\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2) + "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "cmp x16, x19\n" @@ -263,10 +255,10 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "fmla z30.h, p3/M, z4.h, z12.h\n" "csel x16, x16, XZR, LT\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1) + "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" "csel x17, x17, x21, LT\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2) + "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" "cmp x17, x20\n" "fmla z31.h, p3/M, z7.h, z13.h\n" "fmla z30.h, p3/M, z6.h, z13.h\n" @@ -274,24 +266,24 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z29.h, p3/M, z5.h, z10.h\n" "fmla z31.h, p3/M, z2.h, z9.h\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0) + "ld1h { z9.h }, p2/Z, [x26]\n" "fmla z28.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" "fmla z31.h, p3/M, z8.h, z10.h\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2) + "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" "fmla z29.h, p3/M, z7.h, z11.h\n" "fmla z31.h, p3/M, z6.h, z9.h\n" @@ -303,14 +295,14 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "fmla z28.h, p3/M, z7.h, z12.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x28]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) + "st1h { z29.h }, p0, [x22]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1) + "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -321,4 +313,4 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index a845e7c0c6..7cca6fbcbf 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -54,22 +54,22 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( const __fp16 max ) : outptrs(outptrs), params(params), min(min), max(max) { - inptrs[0] = input_ptrs[0]; - inptrs[1] = input_ptrs[1]; - inptrs[2] = input_ptrs[2]; - inptrs[3] = input_ptrs[3]; - inptrs[4] = input_ptrs[4]; - inptrs[5] = input_ptrs[5]; - inptrs[6] = input_ptrs[6]; - inptrs[7] = input_ptrs[7]; - inptrs[8] = input_ptrs[8]; - inptrs[9] = input_ptrs[9]; - inptrs[10] = input_ptrs[10]; - inptrs[11] = input_ptrs[11]; - inptrs[12] = input_ptrs[12]; - inptrs[13] = input_ptrs[13]; - inptrs[14] = input_ptrs[14]; - inptrs[15] = input_ptrs[15]; + inptrs[0] = input_ptrs[5]; + inptrs[1] = input_ptrs[0]; + inptrs[2] = input_ptrs[3]; + inptrs[3] = input_ptrs[6]; + inptrs[4] = input_ptrs[9]; + inptrs[5] = input_ptrs[12]; + inptrs[6] = input_ptrs[15]; + inptrs[7] = input_ptrs[1]; + inptrs[8] = input_ptrs[2]; + inptrs[9] = input_ptrs[10]; + inptrs[10] = input_ptrs[4]; + inptrs[11] = input_ptrs[7]; + inptrs[12] = input_ptrs[8]; + inptrs[13] = input_ptrs[11]; + inptrs[14] = input_ptrs[13]; + inptrs[15] = input_ptrs[14]; } }; @@ -78,207 +78,219 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x3, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x4, [%x[params_struct], %[offsetof_args_params]]\n" - "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x5, #0x0\n" - "ldp x6, x7, [x19, #0x0]\n" - "cnth x8\n" - "ldp x17, x16, [x19, #0x10]\n" - "sub x15, XZR, x8\n" - "ldp x14, x13, [x19, #0x20]\n" - "whilelt p2.h, XZR, %x[n_channels]\n" - "ldp x12, x11, [x19, #0x30]\n" - "cmp x8, %x[n_channels]\n" - "ldp x10, x9, [x19, #0x40]\n" - "ldp x28, x27, [x19, #0x50]\n" - "ldp x26, x25, [x19, #0x60]\n" - "ldp x24, x23, [x19, #0x70]\n" - "ldp x22, x21, [x3, #0x0]\n" - "ldp x20, x19, [x3, #0x10]\n" + "mov x13, #0x0\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias - "addvl x4, x4, #16\n" - "ld1h { z9.h }, p2/Z, [x13, x5, LSL #1]\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias - "addvl x4, x4, #-6\n" - "ld1h { z10.h }, p2/Z, [x6, x5, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x9, x5, LSL #1]\n" + "cnth x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" + "whilelt p2.h, XZR, %x[n_channels]\n" + "ld1h { z16.h }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldr x22, [x14, #0x20]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z4.h, z9.h\n" - "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias - "whilelt p1.h, x8, %x[n_channels]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "inch x15\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" + "ldr x21, [x14, #0x28]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" + "ldr x20, [x14, #0x30]\n" + "inch x9\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" + "ldr x19, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" + "ldr x24, [x14, #0x50]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z12.h\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z13.h\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z7.h, z13.h\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z6.h, z13.h\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x9, x8, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z10.h\n" + "ldp x24, x23, [x14, #0x10]\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias + "ldr x22, [x14, #0x20]\n" "fmla z28.h, p3/M, z2.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z8.h, z10.h\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" + "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n" - "inch x5\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" + "inch x13\n" "fmla z29.h, p3/M, z7.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias - "whilelt p2.h, x5, %x[n_channels]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z31.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x13, x8, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n" "fmla z28.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x16, x8, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x6, x8, LSL #1]\n" - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" - "addvl x4, x4, #16\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x12, x8, LSL #1]\n" - "inch x8\n" + "addvl x15, x15, #16\n" "fmax z31.h, p3/M, z31.h, z18.h\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias - "cmp x8, %x[n_channels]\n" + "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n" + "inch x12\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias - "addvl x4, x4, #-6\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" - "mov z30.d, z16.d\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" - "mov z29.d, z16.d\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" - "mov z28.d, z16.d\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z4.h, z9.h\n" - "inch x15\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" + "ldr x21, [x14, #0x28]\n" + "inch x9\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" + "ldr x20, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" + "ldr x19, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x24, [x14, #0x50]\n" "fmla z31.h, p3/M, z5.h, z12.h\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z13.h\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z7.h, z13.h\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z6.h, z13.h\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" "fmla z31.h, p3/M, z2.h, z9.h\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" "fmla z28.h, p3/M, z2.h, z12.h\n" "fmla z31.h, p3/M, z8.h, z10.h\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n" - "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" + "fmla z30.h, p3/M, z5.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z11.h\n" "fmla z31.h, p3/M, z6.h, z9.h\n" - "fmla z30.h, p3/M, z8.h, z10.h\n" "fmla z28.h, p3/M, z6.h, z11.h\n" + "fmla z30.h, p3/M, z8.h, z10.h\n" "fmla z29.h, p3/M, z8.h, z12.h\n" + "fmla z28.h, p3/M, z7.h, z12.h\n" "fmax z31.h, p3/M, z31.h, z18.h\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmla z28.h, p3/M, z7.h, z12.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index e1f23aae66..af8af1840a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst +class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 0708f578a8..4126cefa34 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,107 +113,98 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "add x11, x14, x22, LSL #1\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x10, x11, x22, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x8]\n" "add x9, x10, x22, LSL #1\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" "add x28, x9, x22, LSL #1\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "add x27, x15, x15\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" "add x26, x27, x15\n" - "mov z27.d, z16.d\n" - "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" "add x25, x26, x15\n" - "mov z26.d, z16.d\n" - "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" "mul x19, x6, x20\n" // offset = tile_i * ld_output_row - "mov z25.d, z16.d\n" - "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" "madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col - "mov z24.d, z16.d\n" + "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" "mul x19, x19, x23\n" // offset *= output_tile_size - "mov z23.d, z16.d\n" - "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x24, x13, x13\n" + "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x23, x12, x20, LSL #1\n" "add x22, x23, x20, LSL #1\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2) - "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (0, 0) + "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x14]\n" "addvl x8, x8, #16\n" - "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4) + "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n" "cmp x16, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" "addvl x8, x8, #-6\n" - "ld1h { z12.h }, p2/Z, [x28]\n" // Load input point (4, 0) - "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2) + "ld1h { z12.h }, p2/Z, [x28]\n" + "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x16, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" "inch x21\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" "inch x17\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" "inch x16\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" + "ld1h { z16.h }, p3/Z, [x8]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4) + "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" "fmla z31.h, p3/M, z5.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z13.h\n" "fmla z27.h, p3/M, z1.h, z13.h\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" "fmla z30.h, p3/M, z6.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z27.h, p3/M, z3.h, z11.h\n" "fmla z25.h, p3/M, z1.h, z11.h\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x11]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4) + "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z5.h, z10.h\n" "fmla z26.h, p3/M, z4.h, z10.h\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0) + "ld1h { z12.h }, p2/Z, [x9]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z2.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1) + "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" @@ -223,173 +214,164 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3) + "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" "addvl x11, x11, #1\n" "fmla z31.h, p3/M, z4.h, z12.h\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2) + "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" "addvl x14, x14, #1\n" "fmla z24.h, p3/M, z8.h, z13.h\n" - "ld1h { z10.h }, p1/Z, [x14]\n" // Load input point (0, 0) + "ld1h { z10.h }, p1/Z, [x14]\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" "addvl x9, x9, #1\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z12.h }, p2/Z, [x10]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" - "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" "addvl x10, x10, #1\n" "fmla z27.h, p3/M, z8.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2) + "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2) + "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" "whilelt p2.h, x17, %x[n_channels]\n" "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" "addvl x28, x28, #1\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" "cmp x16, %x[n_channels]\n" "fmla z25.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x28]\n" // Load input point (4, 0) + "ld1h { z12.h }, p1/Z, [x28]\n" "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z26.h, p3/M, z5.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z23.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4) + "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" "fmla z24.h, p3/M, z7.h, z13.h\n" - "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z25.h, p3/M, z8.h, z13.h\n" "fmax z31.h, p3/M, z31.h, z18.h\n" "fmla z23.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2) + "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x8, x8, #16\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" "addvl x8, x8, #-6\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0) - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x12]\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z16.d\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2) - "mov z29.d, z16.d\n" - "addvl x12, x12, #1\n" + "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" "fmax z27.h, p3/M, z27.h, z18.h\n" "fmax z26.h, p3/M, z26.h, z18.h\n" + "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0) - "mov z28.d, z16.d\n" + "addvl x12, x12, #1\n" + "fmax z25.h, p3/M, z25.h, z18.h\n" + "st1h { z28.h }, p0, [x23]\n" "fmin z27.h, p3/M, z27.h, z17.h\n" - "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1) - "mov z27.d, z16.d\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2) - "mov z26.d, z16.d\n" - "addvl x23, x23, #1\n" - "fmax z25.h, p3/M, z25.h, z18.h\n" + "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" + "fmin z25.h, p3/M, z25.h, z17.h\n" "fmax z24.h, p3/M, z24.h, z18.h\n" + "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" + "addvl x23, x23, #1\n" "fmax z23.h, p3/M, z23.h, z18.h\n" - "fmin z25.h, p3/M, z25.h, z17.h\n" - "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0) - "mov z25.d, z16.d\n" + "st1h { z25.h }, p0, [x22]\n" "fmin z24.h, p3/M, z24.h, z17.h\n" - "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1) - "mov z24.d, z16.d\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2) - "mov z23.d, z16.d\n" + "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" "ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x6, #0x1\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x7, x7, #0x1\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" "cmp x7, x19\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" "csel x7, x7, XZR, LT\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" "csel x6, x6, x21, LT\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" "cmp x6, x20\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4) + "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" "fmla z31.h, p3/M, z5.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z13.h\n" "fmla z27.h, p3/M, z1.h, z13.h\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" "fmla z30.h, p3/M, z6.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z27.h, p3/M, z3.h, z11.h\n" "fmla z25.h, p3/M, z1.h, z11.h\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x11]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4) + "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z5.h, z10.h\n" "fmla z26.h, p3/M, z4.h, z10.h\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0) + "ld1h { z12.h }, p2/Z, [x9]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z2.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1) + "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" @@ -399,36 +381,36 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3) + "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2) + "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z13.h\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z12.h }, p2/Z, [x10]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z13.h\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2) + "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z25.h, p3/M, z0.h, z12.h\n" @@ -442,29 +424,29 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmax z30.h, p3/M, z30.h, z18.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x12]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmax z27.h, p3/M, z27.h, z18.h\n" - "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2) + "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" "fmax z26.h, p3/M, z26.h, z18.h\n" "fmax z25.h, p3/M, z25.h, z18.h\n" "fmax z24.h, p3/M, z24.h, z18.h\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0) + "st1h { z28.h }, p0, [x23]\n" "fmin z27.h, p3/M, z27.h, z17.h\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1) + "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" "fmin z25.h, p3/M, z25.h, z17.h\n" "fmin z24.h, p3/M, z24.h, z17.h\n" - "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2) + "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" "fmax z23.h, p3/M, z23.h, z18.h\n" - "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0) + "st1h { z25.h }, p0, [x22]\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1) - "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2) + "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -475,4 +457,4 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 770576c5da..f79a36b2a3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -87,130 +87,121 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x6, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x7, [%x[params_struct], %[offsetof_args_params]]\n" - "add x8, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x17, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x16\n" - "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias - "sub x15, XZR, x16\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias + "cnth x12\n" + "ld1h { z16.h }, p3/Z, [x15]\n" + "sub x11, XZR, x12\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias - "cmp x16, %x[n_channels]\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias - "mov z27.d, z16.d\n" - "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias - "mov z26.d, z16.d\n" - "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias - "mov z25.d, z16.d\n" - "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #16\n" - "mov z24.d, z16.d\n" - "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias - "mov z23.d, z16.d\n" - "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #-6\n" - "ldp x14, x13, [x8, #0x0]\n" - "ldp x12, x11, [x8, #0x10]\n" - "ldr x10, [x8, #0x20]\n" - "ld1h { z9.h }, p2/Z, [x14, x17, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x13, x17, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x12, x17, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x10, x17, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ldr x26, [x14, #0x20]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x9, [x8, #0x28]\n" - "whilelt p1.h, x16, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x28, [x8, #0x30]\n" - "inch x15\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x27, [x8, #0x38]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" + "inch x11\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" - "ldr x26, [x8, #0x40]\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" - "ldr x22, [x8, #0x48]\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "ldr x21, [x8, #0x50]\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" - "ldr x20, [x8, #0x58]\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" - "ldr x19, [x8, #0x60]\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" - "ldr x25, [x8, #0x68]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" - "ldr x24, [x8, #0x70]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x23, [x8, #0x78]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.h, p3/M, z3.h, z13.h\n" - "ldr x14, [x8, #0x80]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.h, p3/M, z2.h, z13.h\n" - "ldr x13, [x8, #0x88]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.h, p3/M, z1.h, z13.h\n" - "ldr x12, [x8, #0x90]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" - "ldr x11, [x8, #0x98]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.h, p3/M, z6.h, z11.h\n" - "ldr x10, [x8, #0xa0]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.h, p3/M, z4.h, z11.h\n" - "ldr x9, [x8, #0xa8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.h, p3/M, z3.h, z11.h\n" - "ldr x28, [x8, #0xb0]\n" + "ld1h { z16.h }, p3/Z, [x15]\n" "fmla z25.h, p3/M, z1.h, z11.h\n" - "ldr x27, [x8, #0xb8]\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" - "ldr x26, [x8, #0xc0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ldr x22, [x6, #0x0]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.h, p3/M, z5.h, z10.h\n" - "ldr x21, [x6, #0x8]\n" "fmla z26.h, p3/M, z4.h, z10.h\n" - "ldr x20, [x6, #0x10]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" - "ldr x19, [x6, #0x18]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.h, p3/M, z2.h, z10.h\n" - "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z8.h, z10.h\n" @@ -219,191 +210,182 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ldp x14, x13, [x8, #0x0]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" - "ld1h { z9.h }, p1/Z, [x14, x16, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n" - "fmla z30.h, p3/M, z5.h, z11.h\n" - "ld1h { z10.h }, p1/Z, [x13, x16, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ldp x12, x11, [x8, #0x10]\n" + "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z13.h\n" - "ldr x10, [x8, #0x20]\n" + "ldr x26, [x14, #0x20]\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" - "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z13.h\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n" - "inch x17\n" + "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n" + "inch x13\n" "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias - "whilelt p2.h, x17, %x[n_channels]\n" + "ldp x10, x9, [x14, #0x0]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias + "ldp x28, x27, [x14, #0x10]\n" "fmla z25.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x11, x16, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n" "fmla z26.h, p3/M, z5.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n" "fmla z23.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x12, x16, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n" "fmla z25.h, p3/M, z8.h, z13.h\n" - "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n" "fmla z24.h, p3/M, z7.h, z13.h\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" "fmax z31.h, p3/M, z31.h, z18.h\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z23.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x10, x16, LSL #1]\n" - "inch x16\n" + "ld1h { z13.h }, p1/Z, [x26, x12, LSL #1]\n" + "inch x12\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #16\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias - "cmp x16, %x[n_channels]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #-6\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" - "mov z31.d, z16.d\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "fmax z27.h, p3/M, z27.h, z18.h\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "ldr x22, [x6, #0x20]\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" - "mov z30.d, z16.d\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" - "mov z29.d, z16.d\n" - "ldr x21, [x6, #0x28]\n" - "fmax z27.h, p3/M, z27.h, z18.h\n" - "ldr x20, [x6, #0x30]\n" + "ldr x22, [x16, #0x20]\n" "fmax z26.h, p3/M, z26.h, z18.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" - "mov z28.d, z16.d\n" - "ldr x19, [x6, #0x38]\n" - "fmax z25.h, p3/M, z25.h, z18.h\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" + "fmin z28.h, p3/M, z28.h, z17.h\n" "fmin z27.h, p3/M, z27.h, z17.h\n" - "st1h { z27.h }, p0, [x22, x15, LSL #1]\n" - "mov z27.d, z16.d\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "ldr x22, [x6, #0x40]\n" - "fmin z25.h, p3/M, z25.h, z17.h\n" - "st1h { z26.h }, p0, [x21, x15, LSL #1]\n" - "mov z26.d, z16.d\n" + "ldr x21, [x16, #0x28]\n" + "fmax z25.h, p3/M, z25.h, z18.h\n" + "ldr x20, [x16, #0x30]\n" "fmax z24.h, p3/M, z24.h, z18.h\n" - "st1h { z25.h }, p0, [x20, x15, LSL #1]\n" - "mov z25.d, z16.d\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmax z23.h, p3/M, z23.h, z18.h\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" + "fmin z25.h, p3/M, z25.h, z17.h\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.h, p3/M, z24.h, z17.h\n" - "st1h { z24.h }, p0, [x19, x15, LSL #1]\n" - "mov z24.d, z16.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z23.h }, p0, [x22, x15, LSL #1]\n" - "mov z23.d, z16.d\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x9, [x8, #0x28]\n" - "inch x15\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x28, [x8, #0x30]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" + "inch x11\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x27, [x8, #0x38]\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" - "ldr x26, [x8, #0x40]\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" - "ldr x22, [x8, #0x48]\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "ldr x21, [x8, #0x50]\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" - "ldr x20, [x8, #0x58]\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" - "ldr x19, [x8, #0x60]\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" - "ldr x25, [x8, #0x68]\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" - "ldr x24, [x8, #0x70]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x23, [x8, #0x78]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.h, p3/M, z3.h, z13.h\n" - "ldr x14, [x8, #0x80]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.h, p3/M, z2.h, z13.h\n" - "ldr x13, [x8, #0x88]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.h, p3/M, z1.h, z13.h\n" - "ldr x12, [x8, #0x90]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" - "ldr x11, [x8, #0x98]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.h, p3/M, z6.h, z11.h\n" - "ldr x10, [x8, #0xa0]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.h, p3/M, z4.h, z11.h\n" - "ldr x9, [x8, #0xa8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.h, p3/M, z3.h, z11.h\n" - "ldr x28, [x8, #0xb0]\n" "fmla z25.h, p3/M, z1.h, z11.h\n" - "ldr x27, [x8, #0xb8]\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" - "ldr x26, [x8, #0xc0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ldr x22, [x6, #0x0]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.h, p3/M, z5.h, z10.h\n" - "ldr x21, [x6, #0x8]\n" "fmla z26.h, p3/M, z4.h, z10.h\n" - "ldr x20, [x6, #0x10]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" - "ldr x19, [x6, #0x18]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.h, p3/M, z2.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z8.h, z10.h\n" @@ -412,36 +394,36 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z13.h\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z13.h\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z25.h, p3/M, z0.h, z12.h\n" @@ -455,41 +437,41 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "fmax z30.h, p3/M, z30.h, z18.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "ldr x22, [x6, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" "fmax z27.h, p3/M, z27.h, z18.h\n" "fmax z26.h, p3/M, z26.h, z18.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "ldr x21, [x6, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z25.h, p3/M, z25.h, z18.h\n" - "ldr x20, [x6, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmax z24.h, p3/M, z24.h, z18.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z17.h\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "ldr x19, [x6, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z25.h, p3/M, z25.h, z17.h\n" - "st1h { z27.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" "fmin z24.h, p3/M, z24.h, z17.h\n" "fmax z23.h, p3/M, z23.h, z18.h\n" - "st1h { z26.h }, p0, [x21, x15, LSL #1]\n" - "st1h { z25.h }, p0, [x20, x15, LSL #1]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z24.h }, p0, [x19, x15, LSL #1]\n" - "ldr x22, [x6, #0x40]\n" - "st1h { z23.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" + "ldr x22, [x16, #0x40]\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 7d035f0571..60234c8a86 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst +class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 93e1908df7..3cfac06449 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,92 +113,75 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "add x15, x8, x22, LSL #1\n" "ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x14, x15, x22, LSL #1\n" - "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias - "mov z31.d, z13.d\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z13.h }, p3/Z, [x4]\n" "add x13, x14, x22, LSL #1\n" - "mov z30.d, z13.d\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" "add x12, x13, x22, LSL #1\n" - "mov z29.d, z13.d\n" - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" "add x11, x12, x22, LSL #1\n" - "mov z28.d, z13.d\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" "add x10, x7, x7\n" - "mov z27.d, z13.d\n" - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" "add x9, x10, x7\n" - "mov z26.d, z13.d\n" - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" "add x28, x9, x7\n" - "mov z25.d, z13.d\n" - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" "add x27, x28, x7\n" - "mov z24.d, z13.d\n" + "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" "mul x19, x2, x20\n" // offset = tile_i * ld_output_row - "mov z23.d, z13.d\n" + "add x26, x17, x17\n" "madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col - "mov z22.d, z13.d\n" "mul x19, x19, x23\n" // offset *= output_tile_size - "mov z21.d, z13.d\n" "add x16, x16, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "mov z20.d, z13.d\n" - "add x26, x16, x20, LSL #1\n" - "mov z19.d, z13.d\n" - "add x25, x26, x20, LSL #1\n" - "mov z18.d, z13.d\n" + "add x25, x16, x20, LSL #1\n" "add x24, x25, x20, LSL #1\n" - "mov z17.d, z13.d\n" - "add x23, x17, x17\n" - "mov z16.d, z13.d\n" - "add x22, x23, x17\n" + "add x23, x24, x20, LSL #1\n" + "add x22, x26, x17\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2) - "ld1h { z10.h }, p2/Z, [x8]\n" // Load input point (0, 0) + "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x8]\n" "addvl x4, x4, #16\n" - "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5) + "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n" "cmp x6, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" "addvl x4, x4, #-6\n" - "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x6, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" "inch x21\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" "inch x5\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" "inch x6\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2) + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0) - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5) + "ld1h { z10.h }, p2/Z, [x11]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" "fmla z29.h, p3/M, z7.h, z12.h\n" "fmla z26.h, p3/M, z5.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" "fmla z22.h, p3/M, z2.h, z12.h\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1) - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3) - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4) + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" "fmla z26.h, p3/M, z7.h, z9.h\n" "fmla z25.h, p3/M, z6.h, z9.h\n" @@ -206,15 +189,16 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.h, p3/M, z4.h, z9.h\n" "fmla z21.h, p3/M, z3.h, z9.h\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0) + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" + "ld1h { z13.h }, p3/Z, [x4]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5) + "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0) + "ld1h { z11.h }, p2/Z, [x12]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" "fmla z25.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z6.h, z10.h\n" @@ -224,261 +208,245 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2) + "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5) + "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1) + "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1) + "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4) + "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4) + "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2) + "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" "addvl x8, x8, #1\n" "fmla z31.h, p3/M, z2.h, z10.h\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0) + "ld1h { z10.h }, p2/Z, [x14]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5) + "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" "addvl x14, x14, #1\n" "fmla z31.h, p3/M, z6.h, z10.h\n" - "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2) + "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0) + "ld1h { z10.h }, p2/Z, [x13]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2) + "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5) + "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" "addvl x13, x13, #1\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2) + "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3) + "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3) + "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" "addvl x11, x11, #1\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1) + "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4) + "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" "addvl x15, x15, #1\n" "fmla z18.h, p3/M, z8.h, z12.h\n" "fmla z31.h, p3/M, z4.h, z10.h\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1) + "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4) + "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" "whilelt p2.h, x5, %x[n_channels]\n" "fmla z29.h, p3/M, z5.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" "addvl x12, x12, #1\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "cmp x6, %x[n_channels]\n" "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" "fmla z24.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5) + "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" "fmla z23.h, p3/M, z7.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" "fmla z22.h, p3/M, z6.h, z12.h\n" - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" "fmla z19.h, p3/M, z4.h, z12.h\n" "fmla z18.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n" "fmla z21.h, p3/M, z8.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" "fmla z20.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z5.h, z10.h\n" - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" "fmla z16.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x8]\n" // Load input point (0, 0) + "ld1h { z10.h }, p1/Z, [x8]\n" "fmax z31.h, p3/M, z31.h, z15.h\n" - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" "addvl x4, x4, #16\n" "fmax z30.h, p3/M, z30.h, z15.h\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z15.h\n" - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" "addvl x4, x4, #-6\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0) - "mov z31.d, z13.d\n" + "st1h { z31.h }, p0, [x16]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" - "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z13.d\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2) - "mov z29.d, z13.d\n" + "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmax z27.h, p3/M, z27.h, z15.h\n" + "st1h { z29.h }, p0, [x16, x26, LSL #1]\n" "fmax z26.h, p3/M, z26.h, z15.h\n" "fmax z25.h, p3/M, z25.h, z15.h\n" + "fmax z24.h, p3/M, z24.h, z15.h\n" "fmin z28.h, p3/M, z28.h, z14.h\n" - "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3) - "mov z28.d, z13.d\n" - "addvl x16, x16, #1\n" + "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" - "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0) - "mov z27.d, z13.d\n" + "addvl x16, x16, #1\n" "fmin z26.h, p3/M, z26.h, z14.h\n" - "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1) - "mov z26.d, z13.d\n" + "st1h { z27.h }, p0, [x25]\n" "fmin z25.h, p3/M, z25.h, z14.h\n" - "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2) - "mov z25.d, z13.d\n" - "fmax z24.h, p3/M, z24.h, z15.h\n" + "fmin z24.h, p3/M, z24.h, z14.h\n" + "st1h { z26.h }, p0, [x25, x17, LSL #1]\n" "fmax z23.h, p3/M, z23.h, z15.h\n" + "st1h { z25.h }, p0, [x25, x26, LSL #1]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" "fmax z21.h, p3/M, z21.h, z15.h\n" - "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3) - "mov z24.d, z13.d\n" - "addvl x26, x26, #1\n" + "st1h { z24.h }, p0, [x25, x22, LSL #1]\n" + "addvl x25, x25, #1\n" "fmin z23.h, p3/M, z23.h, z14.h\n" - "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0) - "mov z23.d, z13.d\n" + "st1h { z23.h }, p0, [x24]\n" "fmin z22.h, p3/M, z22.h, z14.h\n" - "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1) - "mov z22.d, z13.d\n" "fmin z21.h, p3/M, z21.h, z14.h\n" - "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2) - "mov z21.d, z13.d\n" + "st1h { z22.h }, p0, [x24, x17, LSL #1]\n" "fmax z20.h, p3/M, z20.h, z15.h\n" "fmax z19.h, p3/M, z19.h, z15.h\n" + "st1h { z21.h }, p0, [x24, x26, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" "fmax z17.h, p3/M, z17.h, z15.h\n" + "fmax z16.h, p3/M, z16.h, z15.h\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3) - "mov z20.d, z13.d\n" - "addvl x25, x25, #1\n" + "st1h { z20.h }, p0, [x24, x22, LSL #1]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0) - "mov z19.d, z13.d\n" + "addvl x24, x24, #1\n" "fmin z18.h, p3/M, z18.h, z14.h\n" - "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1) - "mov z18.d, z13.d\n" + "st1h { z19.h }, p0, [x23]\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2) - "mov z17.d, z13.d\n" - "fmax z16.h, p3/M, z16.h, z15.h\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3) - "mov z16.d, z13.d\n" - "addvl x24, x24, #1\n" + "st1h { z18.h }, p0, [x23, x17, LSL #1]\n" + "st1h { z17.h }, p0, [x23, x26, LSL #1]\n" + "st1h { z16.h }, p0, [x23, x22, LSL #1]\n" + "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x2, #0x1\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x3, x3, #0x1\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" "cmp x3, x19\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" "csel x3, x3, XZR, LT\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" "csel x2, x2, x21, LT\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2) + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" "cmp x2, x20\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0) - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5) + "ld1h { z10.h }, p2/Z, [x11]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" "fmla z29.h, p3/M, z7.h, z12.h\n" "fmla z26.h, p3/M, z5.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" "fmla z22.h, p3/M, z2.h, z12.h\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1) - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3) - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4) + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" "fmla z26.h, p3/M, z7.h, z9.h\n" "fmla z25.h, p3/M, z6.h, z9.h\n" @@ -486,15 +454,15 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.h, p3/M, z4.h, z9.h\n" "fmla z21.h, p3/M, z3.h, z9.h\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0) + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5) + "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0) + "ld1h { z11.h }, p2/Z, [x12]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" "fmla z25.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z6.h, z10.h\n" @@ -504,117 +472,117 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2) + "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5) + "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1) + "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1) + "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4) + "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4) + "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2) + "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0) + "ld1h { z10.h }, p2/Z, [x14]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5) + "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0) + "ld1h { z10.h }, p2/Z, [x13]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2) + "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5) + "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2) + "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3) + "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3) + "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1) + "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4) + "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z10.h\n" "fmla z18.h, p3/M, z8.h, z12.h\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1) + "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4) + "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z25.h, p3/M, z2.h, z11.h\n" @@ -632,49 +600,49 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmax z29.h, p3/M, z29.h, z15.h\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x16]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z14.h\n" "fmax z27.h, p3/M, z27.h, z15.h\n" - "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2) + "st1h { z29.h }, p0, [x16, x26, LSL #1]\n" "fmax z26.h, p3/M, z26.h, z15.h\n" - "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3) + "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" "fmax z25.h, p3/M, z25.h, z15.h\n" - "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0) + "st1h { z27.h }, p0, [x25]\n" "fmin z26.h, p3/M, z26.h, z14.h\n" "fmin z25.h, p3/M, z25.h, z14.h\n" - "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1) + "st1h { z26.h }, p0, [x25, x17, LSL #1]\n" "fmax z24.h, p3/M, z24.h, z15.h\n" "fmax z23.h, p3/M, z23.h, z15.h\n" - "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2) + "st1h { z25.h }, p0, [x25, x26, LSL #1]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" "fmax z21.h, p3/M, z21.h, z15.h\n" "fmax z20.h, p3/M, z20.h, z15.h\n" "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3) + "st1h { z24.h }, p0, [x25, x22, LSL #1]\n" "fmin z23.h, p3/M, z23.h, z14.h\n" "fmin z22.h, p3/M, z22.h, z14.h\n" - "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0) + "st1h { z23.h }, p0, [x24]\n" "fmin z21.h, p3/M, z21.h, z14.h\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1) + "st1h { z22.h }, p0, [x24, x17, LSL #1]\n" "fmax z19.h, p3/M, z19.h, z15.h\n" - "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2) + "st1h { z21.h }, p0, [x24, x26, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" "fmax z17.h, p3/M, z17.h, z15.h\n" - "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3) + "st1h { z20.h }, p0, [x24, x22, LSL #1]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0) + "st1h { z19.h }, p0, [x23]\n" "fmin z18.h, p3/M, z18.h, z14.h\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1) + "st1h { z18.h }, p0, [x23, x17, LSL #1]\n" "fmax z16.h, p3/M, z16.h, z15.h\n" - "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2) + "st1h { z17.h }, p0, [x23, x26, LSL #1]\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3) + "st1h { z16.h }, p0, [x23, x22, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -685,4 +653,4 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 8eaf0a46d6..66f6c3bb7a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -98,570 +98,538 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n" - "add x4, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z15.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x5, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x6\n" - "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias - "mov z31.d, z13.d\n" - "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias - "sub x7, XZR, x6\n" - "mov z30.d, z13.d\n" - "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias + "cnth x12\n" + "ld1h { z13.h }, p3/Z, [x15]\n" + "sub x11, XZR, x12\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "mov z29.d, z13.d\n" - "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias - "cmp x6, %x[n_channels]\n" - "mov z28.d, z13.d\n" - "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias - "mov z27.d, z13.d\n" - "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias - "mov z26.d, z13.d\n" - "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias - "mov z25.d, z13.d\n" - "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #16\n" - "mov z24.d, z13.d\n" - "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias - "mov z23.d, z13.d\n" - "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #-6\n" - "mov z22.d, z13.d\n" - "ldp x8, x17, [x4, #0x0]\n" - "mov z21.d, z13.d\n" - "ldp x16, x15, [x4, #0x10]\n" - "mov z20.d, z13.d\n" - "ld1h { z9.h }, p2/Z, [x8, x5, LSL #1]\n" - "mov z19.d, z13.d\n" - "mov z18.d, z13.d\n" - "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n" - "mov z17.d, z13.d\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" - "mov z16.d, z13.d\n" - "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0x20]\n" - "whilelt p1.h, x6, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0x28]\n" - "inch x7\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0x30]\n" + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x20]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" + "inch x11\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0x38]\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0x40]\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0x48]\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" - "ldr x28, [x4, #0x50]\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" - "ldr x27, [x4, #0x58]\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" - "ldr x26, [x4, #0x60]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.h, p3/M, z7.h, z12.h\n" - "ldr x25, [x4, #0x68]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.h, p3/M, z5.h, z12.h\n" - "ldr x24, [x4, #0x70]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ldr x23, [x4, #0x78]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "ldr x8, [x4, #0x80]\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "ldr x17, [x4, #0x88]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.h, p3/M, z2.h, z12.h\n" - "ldr x16, [x4, #0x90]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "ldr x15, [x4, #0x98]\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n" - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0xa0]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0xa8]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0xb0]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0xb8]\n" "fmla z22.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0xc0]\n" "fmla z21.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0xc8]\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "ldr x22, [x2, #0x0]\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x2, #0x8]\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n" + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" - "ldr x28, [x4, #0xd0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ldr x27, [x4, #0xd8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" - "ldr x26, [x4, #0xe0]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.h, p3/M, z7.h, z10.h\n" - "ldr x20, [x2, #0x10]\n" + "ld1h { z13.h }, p3/Z, [x15]\n" "fmla z24.h, p3/M, z6.h, z10.h\n" - "ldr x19, [x2, #0x18]\n" "fmla z22.h, p3/M, z5.h, z10.h\n" - "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias "fmla z21.h, p3/M, z4.h, z10.h\n" "fmla z20.h, p3/M, z3.h, z10.h\n" "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ldr x25, [x4, #0xe8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" - "ldr x23, [x4, #0xf8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" - "ldr x24, [x4, #0xf0]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" - "ldr x17, [x4, #0x108]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ldr x8, [x4, #0x100]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" - "ldr x15, [x4, #0x118]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" - "ldr x16, [x4, #0x110]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z10.h\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z10.h\n" - "ldp x8, x17, [x4, #0x0]\n" + "ldp x10, x9, [x14, #0x0]\n" "fmla z18.h, p3/M, z8.h, z12.h\n" - "ld1h { z9.h }, p1/Z, [x8, x6, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n" - "inch x5\n" + "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n" + "inch x13\n" "fmla z29.h, p3/M, z5.h, z11.h\n" - "ldp x16, x15, [x4, #0x10]\n" - "whilelt p2.h, x5, %x[n_channels]\n" + "ldp x28, x27, [x14, #0x10]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z28.h, p3/M, z4.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z24.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x16, x6, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n" "fmla z23.h, p3/M, z7.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z22.h, p3/M, z6.h, z12.h\n" - "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" "fmla z19.h, p3/M, z4.h, z12.h\n" "fmla z18.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x15, x6, LSL #1]\n" + "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n" "fmla z21.h, p3/M, z8.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z20.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z5.h, z10.h\n" - "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z16.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x17, x6, LSL #1]\n" - "inch x6\n" + "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n" + "inch x12\n" "fmax z31.h, p3/M, z31.h, z15.h\n" - "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #16\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z30.h, p3/M, z30.h, z15.h\n" - "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias - "cmp x6, %x[n_channels]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.h, p3/M, z29.h, z15.h\n" - "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #-6\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmax z27.h, p3/M, z27.h, z15.h\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x22, x7, LSL #1]\n" - "mov z31.d, z13.d\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" - "ldr x22, [x2, #0x20]\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "st1h { z30.h }, p0, [x21, x7, LSL #1]\n" - "mov z30.d, z13.d\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.h, p3/M, z28.h, z14.h\n" - "st1h { z29.h }, p0, [x20, x7, LSL #1]\n" - "mov z29.d, z13.d\n" - "ldr x21, [x2, #0x28]\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" - "ldr x20, [x2, #0x30]\n" "fmax z26.h, p3/M, z26.h, z15.h\n" - "st1h { z28.h }, p0, [x19, x7, LSL #1]\n" - "mov z28.d, z13.d\n" - "ldr x19, [x2, #0x38]\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmax z25.h, p3/M, z25.h, z15.h\n" - "st1h { z27.h }, p0, [x22, x7, LSL #1]\n" - "mov z27.d, z13.d\n" - "ldr x22, [x2, #0x40]\n" - "fmin z26.h, p3/M, z26.h, z14.h\n" - "st1h { z26.h }, p0, [x21, x7, LSL #1]\n" - "mov z26.d, z13.d\n" - "fmin z25.h, p3/M, z25.h, z14.h\n" - "ldr x21, [x2, #0x48]\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmax z24.h, p3/M, z24.h, z15.h\n" - "st1h { z25.h }, p0, [x20, x7, LSL #1]\n" - "mov z25.d, z13.d\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.h, p3/M, z23.h, z15.h\n" - "ldr x20, [x2, #0x50]\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" + "fmin z26.h, p3/M, z26.h, z14.h\n" + "ldr x20, [x16, #0x30]\n" + "fmin z25.h, p3/M, z25.h, z14.h\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z24.h }, p0, [x19, x7, LSL #1]\n" - "mov z24.d, z13.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.h, p3/M, z23.h, z14.h\n" - "ldr x19, [x2, #0x58]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" - "st1h { z23.h }, p0, [x22, x7, LSL #1]\n" - "mov z23.d, z13.d\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" "fmax z21.h, p3/M, z21.h, z15.h\n" - "ldr x22, [x2, #0x60]\n" - "fmin z22.h, p3/M, z22.h, z14.h\n" - "st1h { z22.h }, p0, [x21, x7, LSL #1]\n" - "mov z22.d, z13.d\n" - "fmin z21.h, p3/M, z21.h, z14.h\n" - "ldr x21, [x2, #0x68]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" "fmax z20.h, p3/M, z20.h, z15.h\n" - "st1h { z21.h }, p0, [x20, x7, LSL #1]\n" - "mov z21.d, z13.d\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" "fmax z19.h, p3/M, z19.h, z15.h\n" - "ldr x20, [x2, #0x70]\n" + "ldr x21, [x16, #0x48]\n" + "fmin z22.h, p3/M, z22.h, z14.h\n" + "ldr x20, [x16, #0x50]\n" + "fmin z21.h, p3/M, z21.h, z14.h\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "st1h { z20.h }, p0, [x19, x7, LSL #1]\n" - "mov z20.d, z13.d\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "ldr x19, [x2, #0x78]\n" + "st1h { z22.h }, p0, [x21, x11, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" - "st1h { z19.h }, p0, [x22, x7, LSL #1]\n" - "mov z19.d, z13.d\n" + "st1h { z21.h }, p0, [x20, x11, LSL #1]\n" "fmax z17.h, p3/M, z17.h, z15.h\n" + "st1h { z20.h }, p0, [x19, x11, LSL #1]\n" + "fmax z16.h, p3/M, z16.h, z15.h\n" + "st1h { z19.h }, p0, [x22, x11, LSL #1]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.h, p3/M, z18.h, z14.h\n" - "st1h { z18.h }, p0, [x21, x7, LSL #1]\n" - "mov z18.d, z13.d\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "st1h { z17.h }, p0, [x20, x7, LSL #1]\n" - "mov z17.d, z13.d\n" - "fmax z16.h, p3/M, z16.h, z15.h\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z16.h }, p0, [x19, x7, LSL #1]\n" - "mov z16.d, z13.d\n" + "st1h { z18.h }, p0, [x21, x11, LSL #1]\n" + "st1h { z17.h }, p0, [x20, x11, LSL #1]\n" + "st1h { z16.h }, p0, [x19, x11, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0x20]\n" - "inch x7\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0x28]\n" + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x20]\n" + "inch x11\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0x30]\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0x38]\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0x40]\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0x48]\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" - "ldr x28, [x4, #0x50]\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" - "ldr x27, [x4, #0x58]\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" - "ldr x26, [x4, #0x60]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.h, p3/M, z7.h, z12.h\n" - "ldr x25, [x4, #0x68]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.h, p3/M, z5.h, z12.h\n" - "ldr x24, [x4, #0x70]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ldr x23, [x4, #0x78]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "ldr x8, [x4, #0x80]\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "ldr x17, [x4, #0x88]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.h, p3/M, z2.h, z12.h\n" - "ldr x16, [x4, #0x90]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "ldr x15, [x4, #0x98]\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n" - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0xa0]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0xa8]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0xb0]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0xb8]\n" "fmla z22.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0xc0]\n" "fmla z21.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0xc8]\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "ldr x22, [x2, #0x0]\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x2, #0x8]\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n" + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" - "ldr x28, [x4, #0xd0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ldr x27, [x4, #0xd8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" - "ldr x26, [x4, #0xe0]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.h, p3/M, z7.h, z10.h\n" - "ldr x20, [x2, #0x10]\n" "fmla z24.h, p3/M, z6.h, z10.h\n" - "ldr x19, [x2, #0x18]\n" "fmla z22.h, p3/M, z5.h, z10.h\n" "fmla z21.h, p3/M, z4.h, z10.h\n" "fmla z20.h, p3/M, z3.h, z10.h\n" "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ldr x25, [x4, #0xe8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" - "ldr x23, [x4, #0xf8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" - "ldr x24, [x4, #0xf0]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" - "ldr x17, [x4, #0x108]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ldr x8, [x4, #0x100]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" - "ldr x15, [x4, #0x118]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" - "ldr x16, [x4, #0x110]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z10.h\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z10.h\n" "fmla z18.h, p3/M, z8.h, z12.h\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z25.h, p3/M, z2.h, z11.h\n" @@ -679,68 +647,68 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "fmax z29.h, p3/M, z29.h, z15.h\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x22, x7, LSL #1]\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "ldr x22, [x2, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.h, p3/M, z28.h, z14.h\n" - "st1h { z30.h }, p0, [x21, x7, LSL #1]\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" "fmax z27.h, p3/M, z27.h, z15.h\n" "fmax z26.h, p3/M, z26.h, z15.h\n" - "st1h { z29.h }, p0, [x20, x7, LSL #1]\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmax z25.h, p3/M, z25.h, z15.h\n" - "st1h { z28.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmax z24.h, p3/M, z24.h, z15.h\n" - "ldr x21, [x2, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.h, p3/M, z23.h, z15.h\n" - "ldr x20, [x2, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" - "ldr x19, [x2, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z26.h, p3/M, z26.h, z14.h\n" - "st1h { z27.h }, p0, [x22, x7, LSL #1]\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" "fmin z25.h, p3/M, z25.h, z14.h\n" "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z26.h }, p0, [x21, x7, LSL #1]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" "fmin z23.h, p3/M, z23.h, z14.h\n" - "ldr x22, [x2, #0x40]\n" + "ldr x22, [x16, #0x40]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" - "ldr x21, [x2, #0x48]\n" + "ldr x21, [x16, #0x48]\n" "fmax z21.h, p3/M, z21.h, z15.h\n" - "st1h { z25.h }, p0, [x20, x7, LSL #1]\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" "fmax z20.h, p3/M, z20.h, z15.h\n" - "st1h { z24.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" "fmax z19.h, p3/M, z19.h, z15.h\n" - "st1h { z23.h }, p0, [x22, x7, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" "fmin z22.h, p3/M, z22.h, z14.h\n" - "ldr x20, [x2, #0x50]\n" + "ldr x20, [x16, #0x50]\n" "fmin z21.h, p3/M, z21.h, z14.h\n" - "ldr x19, [x2, #0x58]\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "ldr x22, [x2, #0x60]\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "st1h { z22.h }, p0, [x21, x7, LSL #1]\n" + "st1h { z22.h }, p0, [x21, x11, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" - "st1h { z21.h }, p0, [x20, x7, LSL #1]\n" + "st1h { z21.h }, p0, [x20, x11, LSL #1]\n" "fmax z17.h, p3/M, z17.h, z15.h\n" - "st1h { z20.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z20.h }, p0, [x19, x11, LSL #1]\n" "fmax z16.h, p3/M, z16.h, z15.h\n" - "st1h { z19.h }, p0, [x22, x7, LSL #1]\n" - "ldr x21, [x2, #0x68]\n" + "st1h { z19.h }, p0, [x22, x11, LSL #1]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.h, p3/M, z18.h, z14.h\n" - "ldr x20, [x2, #0x70]\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "ldr x19, [x2, #0x78]\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z18.h }, p0, [x21, x7, LSL #1]\n" - "st1h { z17.h }, p0, [x20, x7, LSL #1]\n" - "st1h { z16.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z18.h }, p0, [x21, x11, LSL #1]\n" + "st1h { z17.h }, p0, [x20, x11, LSL #1]\n" + "st1h { z16.h }, p0, [x19, x11, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index a6a4afb3b5..5968309927 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst +class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 2238bf08cd..d20e9913ae 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,225 +113,217 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x9, x21, x22, LSL #1\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x28, x9, x22, LSL #1\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias - "mov z31.d, z17.d\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z17.h }, p3/Z, [x16]\n" "add x27, x28, x22, LSL #1\n" - "mov z30.d, z17.d\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "add x26, x27, x22, LSL #1\n" - "mov z29.d, z17.d\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" "add x25, x13, x13\n" - "mov z28.d, z17.d\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" "add x24, x25, x13\n" - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" "add x23, x24, x13\n" - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" "mul x19, x7, x20\n" // offset = tile_i * ld_output_row - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" "madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col - "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2) - "ld1h { z10.h }, p2/Z, [x21]\n" // Load input point (0, 0) + "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "mul x19, x19, x17\n" // offset *= output_tile_size - "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1) + "whilelt p2.h, XZR, %x[n_channels]\n" + "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21]\n" "add x10, x10, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3) + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "add x22, x10, x20, LSL #1\n" - "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4) + "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n" "addvl x16, x16, #16\n" - "ld1h { z14.h }, p2/Z, [x9]\n" // Load input point (1, 0) + "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n" "cmp x14, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" - "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1) - "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2) + "ld1h { z14.h }, p2/Z, [x9]\n" + "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x14, %x[n_channels]\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" "inch x12\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z17.h }, p3/Z, [x16]\n" "inch x15\n" - "addvl x21, x21, #1\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x21]\n" // Load input point (0, 0) + "addvl x21, x21, #1\n" + "ld1h { z10.h }, p1/Z, [x21]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4) "inch x14\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2) - "addvl x9, x9, #1\n" + "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0) + "ld1h { z14.h }, p2/Z, [x27]\n" + "addvl x9, x9, #1\n" "fmla z30.h, p3/M, z0.h, z16.h\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4) + "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0) + "ld1h { z15.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1) + "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" "addvl x28, x28, #1\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2) + "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" "fmla z31.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0) + "ld1h { z15.h }, p2/Z, [x26]\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2) + "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2) + "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" "addvl x27, x27, #1\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z11.h\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4) + "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n" "fmax z31.h, p3/M, z31.h, z19.h\n" "fmla z28.h, p3/M, z5.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3) + "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z19.h\n" "fmla z29.h, p3/M, z5.h, z16.h\n" - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" "fmin z31.h, p3/M, z31.h, z18.h\n" - "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0) - "mov z31.d, z17.d\n" + "st1h { z31.h }, p0, [x10]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4) - "whilelt p2.h, x15, %x[n_channels]\n" "fmla z29.h, p3/M, z8.h, z15.h\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias - "addvl x26, x26, #1\n" + "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" + "whilelt p2.h, x15, %x[n_channels]\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z17.d\n" - "addvl x10, x10, #1\n" + "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" + "addvl x26, x26, #1\n" "fmla z28.h, p3/M, z3.h, z16.h\n" - "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2) + "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n" "cmp x14, %x[n_channels]\n" "fmax z29.h, p3/M, z29.h, z19.h\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" + "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z14.h\n" - "ld1h { z14.h }, p1/Z, [x9]\n" // Load input point (1, 0) + "ld1h { z14.h }, p1/Z, [x9]\n" + "addvl x10, x10, #1\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "mov z29.d, z17.d\n" + "st1h { z29.h }, p0, [x22]\n" "fmla z28.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1) - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1) - "addvl x16, x16, #16\n" + "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n" + "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "fmax z28.h, p3/M, z28.h, z19.h\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias + "addvl x16, x16, #16\n" + "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1) - "mov z28.d, z17.d\n" - "addvl x22, x22, #1\n" + "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" + "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" + "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x7, #0x1\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x8, x8, #0x1\n" "fmla z31.h, p3/M, z0.h, z10.h\n" "cmp x8, x19\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4) + "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" "csel x8, x8, XZR, LT\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2) + "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" "csel x7, x7, x21, LT\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0) + "ld1h { z14.h }, p2/Z, [x27]\n" "cmp x7, x20\n" "fmla z30.h, p3/M, z0.h, z16.h\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4) + "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0) + "ld1h { z15.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1) + "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" "fmla z31.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0) + "ld1h { z15.h }, p2/Z, [x26]\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2) + "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2) + "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z8.h, z11.h\n" "fmla z29.h, p3/M, z7.h, z13.h\n" "fmax z31.h, p3/M, z31.h, z19.h\n" "fmla z28.h, p3/M, z5.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3) + "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z19.h\n" "fmla z29.h, p3/M, z5.h, z16.h\n" "fmin z31.h, p3/M, z31.h, z18.h\n" - "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x10]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z8.h, z15.h\n" - "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4) + "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z16.h\n" "fmax z29.h, p3/M, z29.h, z19.h\n" "fmla z28.h, p3/M, z7.h, z14.h\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) + "st1h { z29.h }, p0, [x22]\n" "fmla z28.h, p3/M, z6.h, z15.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" "fmax z28.h, p3/M, z28.h, z19.h\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1) + "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -342,4 +334,4 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 0d5d4176aa..ceba36d897 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -89,257 +89,249 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" - "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z19.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x14, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x13\n" - "ldp x12, x11, [x19, #0x0]\n" - "sub x10, XZR, x13\n" - "ldp x9, x28, [x19, #0x10]\n" + "cnth x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias - "mov z31.d, z17.d\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias - "cmp x13, %x[n_channels]\n" - "mov z30.d, z17.d\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias - "mov z29.d, z17.d\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias - "mov z28.d, z17.d\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #16\n" - "ldp x27, x26, [x15, #0x0]\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #-6\n" - "ld1h { z9.h }, p2/Z, [x27, x14, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldp x25, x23, [x15, #0x10]\n" - "ldp x22, x21, [x15, #0x20]\n" - "ldp x20, x19, [x15, #0x30]\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x23, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x21, x14, LSL #1]\n" - "ld1h { z15.h }, p2/Z, [x20, x14, LSL #1]\n" - "ld1h { z16.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z17.h }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x21, x13, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x20, x13, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x19, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x24, [x15, #0x40]\n" - "whilelt p1.h, x13, %x[n_channels]\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "ldr x20, [x15, #0x48]\n" - "inch x10\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" - "ldr x23, [x15, #0x50]\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x40]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" + "ldr x25, [x14, #0x48]\n" + "inch x9\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" + "ldr x24, [x14, #0x50]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ldr x19, [x15, #0x58]\n" - "ldr x22, [x15, #0x60]\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ldr x21, [x15, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x20, [x15, #0x70]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n" - "ldr x19, [x15, #0x78]\n" + "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.h, p3/M, z0.h, z16.h\n" - "ldr x27, [x15, #0x80]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ldr x26, [x15, #0x88]\n" - "ldr x25, [x15, #0x90]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" - "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x23, [x15, #0x98]\n" + "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n" - "ldr x22, [x15, #0xa0]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x21, [x15, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" - "ldr x20, [x15, #0xb0]\n" - "ldr x19, [x15, #0xb8]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.h, p3/M, z6.h, z15.h\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n" - "ldr x24, [x15, #0xc0]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ldp x27, x26, [x15, #0x0]\n" + "ld1h { z17.h }, p3/Z, [x15]\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z11.h\n" - "ldp x25, x23, [x15, #0x10]\n" - "ldp x22, x21, [x15, #0x20]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z28.h, p3/M, z5.h, z14.h\n" "fmax z31.h, p3/M, z31.h, z19.h\n" - "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x27, x13, LSL #1]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" "fmax z30.h, p3/M, z30.h, z19.h\n" - "ld1h { z10.h }, p1/Z, [x26, x13, LSL #1]\n" - "ld1h { z12.h }, p1/Z, [x23, x13, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "fmin z31.h, p3/M, z31.h, z18.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" - "inch x14\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" + "inch x13\n" "fmla z29.h, p3/M, z5.h, z16.h\n" - "ld1h { z13.h }, p1/Z, [x22, x13, LSL #1]\n" - "whilelt p2.h, x14, %x[n_channels]\n" + "ldp x26, x25, [x14, #0x0]\n" + "whilelt p2.h, x13, %x[n_channels]\n" + "fmin z31.h, p3/M, z31.h, z18.h\n" + "ldp x24, x23, [x14, #0x10]\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "ldp x20, x19, [x15, #0x30]\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" "fmla z28.h, p3/M, z3.h, z16.h\n" - "st1h { z31.h }, p0, [x12, x10, LSL #1]\n" - "mov z31.d, z17.d\n" - "ld1h { z16.h }, p1/Z, [x19, x13, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z15.h\n" - "st1h { z30.h }, p0, [x11, x10, LSL #1]\n" - "mov z30.d, z17.d\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z14.h\n" - "ld1h { z14.h }, p1/Z, [x21, x13, LSL #1]\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n" + "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n" "fmax z29.h, p3/M, z29.h, z19.h\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z14.h }, p1/Z, [x21, x12, LSL #1]\n" "fmla z28.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p1/Z, [x20, x13, LSL #1]\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z15.h }, p1/Z, [x20, x12, LSL #1]\n" + "ld1h { z16.h }, p1/Z, [x19, x12, LSL #1]\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x25, x13, LSL #1]\n" - "inch x13\n" + "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n" + "inch x12\n" "fmax z28.h, p3/M, z28.h, z19.h\n" - "st1h { z29.h }, p0, [x9, x10, LSL #1]\n" - "cmp x13, %x[n_channels]\n" - "mov z29.d, z17.d\n" - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #16\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" + "cmp x12, %x[n_channels]\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #-6\n" - "st1h { z28.h }, p0, [x28, x10, LSL #1]\n" - "mov z28.d, z17.d\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x24, [x15, #0x40]\n" - "inch x10\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "ldr x20, [x15, #0x48]\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x40]\n" + "inch x9\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" + "ldr x25, [x14, #0x48]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" - "ldr x23, [x15, #0x50]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ldr x19, [x15, #0x58]\n" - "ldr x22, [x15, #0x60]\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" + "ldr x24, [x14, #0x50]\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ldr x21, [x15, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x20, [x15, #0x70]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n" - "ldr x19, [x15, #0x78]\n" + "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.h, p3/M, z0.h, z16.h\n" - "ldr x27, [x15, #0x80]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ldr x26, [x15, #0x88]\n" - "ldr x25, [x15, #0x90]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" - "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x23, [x15, #0x98]\n" + "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n" - "ldr x22, [x15, #0xa0]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x21, [x15, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" - "ldr x20, [x15, #0xb0]\n" - "ldr x19, [x15, #0xb8]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.h, p3/M, z6.h, z15.h\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n" - "ldr x24, [x15, #0xc0]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z11.h\n" "fmla z28.h, p3/M, z5.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n" "fmax z31.h, p3/M, z31.h, z19.h\n" "fmla z29.h, p3/M, z7.h, z13.h\n" "fmax z30.h, p3/M, z30.h, z19.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmin z31.h, p3/M, z31.h, z18.h\n" - "st1h { z31.h }, p0, [x12, x10, LSL #1]\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z16.h\n" "fmla z28.h, p3/M, z3.h, z16.h\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "st1h { z30.h }, p0, [x11, x10, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z14.h\n" "fmla z29.h, p3/M, z8.h, z15.h\n" "fmla z28.h, p3/M, z6.h, z15.h\n" "fmax z29.h, p3/M, z29.h, z19.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "st1h { z29.h }, p0, [x9, x10, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z19.h\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "st1h { z28.h }, p0, [x28, x10, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 236f9bf43a..4a9bd33a1e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst +class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const __fp16 *const *>(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const __fp16 *>(activation_min), + *static_cast<const __fp16 *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index 6b1564e6c9..1c2e1e27ad 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,390 +113,382 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "add x20, x13, x22, LSL #1\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x10, x20, x22, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x8]\n" "add x9, x10, x22, LSL #1\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" "add x28, x9, x22, LSL #1\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "add x27, x28, x22, LSL #1\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" "add x26, x15, x15\n" - "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" "add x25, x26, x15\n" - "mul x19, x5, x21\n" // offset = tile_i * ld_output_row + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" "add x24, x25, x15\n" + "mul x19, x5, x21\n" // offset = tile_i * ld_output_row "add x23, x24, x15\n" "madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col "mul x19, x19, x7\n" // offset *= output_tile_size "add x11, x11, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x22, x11, x21, LSL #1\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z5.h }, p2/Z, [x13]\n" // Load input point (0, 0) - "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z5.h }, p2/Z, [x13]\n" + "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n" "cmp x16, %x[n_channels]\n" - "ld1h { z7.h }, p2/Z, [x20]\n" // Load input point (1, 0) + "ld1h { z7.h }, p2/Z, [x20]\n" "addvl x8, x8, #6\n" - "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1) - "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2) - "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2) - "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3) - "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4) - "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5) - "ld1h { z14.h }, p2/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x10]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3) + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" "whilelt p1.h, x16, %x[n_channels]\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" "inch x14\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z0.h }, p3/Z, [x8]\n" "inch x17\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4) + "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" "addvl x20, x20, #1\n" "fmla z30.h, p3/M, z1.h, z9.h\n" "inch x16\n" "fmla z29.h, p3/M, z1.h, z8.h\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5) + "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" "addvl x13, x13, #1\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z5.h\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2) + "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3) + "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" - "ld1h { z7.h }, p1/Z, [x20]\n" // Load input point (1, 0) + "ld1h { z7.h }, p1/Z, [x20]\n" "fmla z30.h, p3/M, z0.h, z8.h\n" "fmla z29.h, p3/M, z0.h, z14.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5) + "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" "fmla z29.h, p3/M, z1.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4) + "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" "addvl x10, x10, #1\n" "fmla z30.h, p3/M, z2.h, z5.h\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x8, x8, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0) - "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p2/Z, [x9]\n" + "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5) + "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z5.h\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z6.h\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4) + "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" "addvl x9, x9, #1\n" "fmla z30.h, p3/M, z2.h, z9.h\n" "fmla z29.h, p3/M, z2.h, z10.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0) + "ld1h { z9.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4) + "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2) + "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3) + "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5) + "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" "addvl x28, x28, #1\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1) + "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" - "ld1h { z14.h }, p1/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z14.h }, p1/Z, [x10]\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2) + "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3) + "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2) + "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4) + "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" - "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z5.h\n" - "ld1h { z5.h }, p1/Z, [x13]\n" // Load input point (0, 0) + "ld1h { z5.h }, p1/Z, [x13]\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5) + "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" "whilelt p2.h, x17, %x[n_channels]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x27, x27, #1\n" "fmla z31.h, p3/M, z3.h, z6.h\n" - "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n" "addvl x8, x8, #16\n" "fmla z30.h, p3/M, z3.h, z8.h\n" "cmp x16, %x[n_channels]\n" "fmla z29.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3) + "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1) + "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5) + "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4) + "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n" "fmla z28.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2) - "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n" + "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" "fmax z31.h, p3/M, z31.h, z18.h\n" "addvl x8, x8, #-6\n" "fmax z30.h, p3/M, z30.h, z18.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0) - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x11]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z16.d\n" - "addvl x11, x11, #1\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "mov z29.d, z16.d\n" + "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1) - "mov z28.d, z16.d\n" + "addvl x11, x11, #1\n" + "st1h { z29.h }, p0, [x22]\n" + "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3) + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" "add x21, x5, #0x1\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z0.h }, p3/Z, [x8]\n" "add x6, x6, #0x1\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4) + "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmla z29.h, p3/M, z1.h, z8.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "cmp x6, x19\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5) + "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" "csel x6, x6, XZR, LT\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "csel x5, x5, x21, LT\n" "fmla z29.h, p3/M, z2.h, z13.h\n" "cmp x5, x20\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z5.h\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2) + "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3) + "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" "fmla z30.h, p3/M, z0.h, z8.h\n" "fmla z29.h, p3/M, z0.h, z14.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5) + "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" "fmla z29.h, p3/M, z1.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4) + "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z5.h\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x8, x8, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0) + "ld1h { z5.h }, p2/Z, [x9]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5) + "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z5.h\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z6.h\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4) + "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z9.h\n" "fmla z29.h, p3/M, z2.h, z10.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0) + "ld1h { z9.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4) + "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2) + "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3) + "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5) + "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1) + "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2) + "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3) + "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4) + "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" "fmla z31.h, p3/M, z2.h, z5.h\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5) + "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" "fmla z31.h, p3/M, z3.h, z6.h\n" "fmla z30.h, p3/M, z3.h, z8.h\n" @@ -511,13 +503,13 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "fmax z29.h, p3/M, z29.h, z18.h\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x11]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1) + "st1h { z29.h }, p0, [x22]\n" + "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -528,4 +520,4 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index be128b4aff..a0640daeca 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -100,432 +100,424 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x5, [%x[params_struct], %[offsetof_args_params]]\n" - "add x6, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x7, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x8\n" - "ldp x17, x16, [x19, #0x0]\n" - "sub x15, XZR, x8\n" - "ldp x14, x13, [x19, #0x10]\n" + "cnth x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z16.h }, p3/Z, [x5]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias - "cmp x8, %x[n_channels]\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias - "ld1h { z4.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias - "addvl x5, x5, #6\n" - "ldp x12, x11, [x6, #0x0]\n" - "ldp x10, x9, [x6, #0x10]\n" - "ldp x20, x28, [x6, #0x20]\n" - "ld1h { z5.h }, p2/Z, [x12, x7, LSL #1]\n" - "ld1h { z6.h }, p2/Z, [x11, x7, LSL #1]\n" - "ld1h { z7.h }, p2/Z, [x10, x7, LSL #1]\n" - "ld1h { z8.h }, p2/Z, [x9, x7, LSL #1]\n" - "ld1h { z9.h }, p2/Z, [x20, x7, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x28, x7, LSL #1]\n" - "ldp x27, x19, [x6, #0x30]\n" - "ldp x26, x25, [x6, #0x40]\n" - "ld1h { z11.h }, p2/Z, [x27, x7, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x26, x7, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x25, x7, LSL #1]\n" + "ld1h { z16.h }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #6\n" + "ldp x26, x25, [x14, #0x0]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ld1h { z5.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x25, x13, LSL #1]\n" + "ld1h { z7.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x23, x13, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ldp x26, x25, [x14, #0x40]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ldr x24, [x6, #0x50]\n" - "whilelt p1.h, x8, %x[n_channels]\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" - "ldr x23, [x6, #0x58]\n" - "inch x15\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" - "ldr x22, [x6, #0x60]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ldr x24, [x14, #0x50]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" + "ldr x23, [x14, #0x58]\n" + "inch x9\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" + "ldr x22, [x14, #0x60]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x6, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.h, p3/M, z1.h, z8.h\n" - "ldr x20, [x6, #0x70]\n" + "ldr x20, [x14, #0x70]\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ldr x19, [x6, #0x78]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.h, p3/M, z2.h, z13.h\n" - "ldr x12, [x6, #0x80]\n" + "ldr x26, [x14, #0x80]\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" - "ldr x11, [x6, #0x88]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.h, p3/M, z3.h, z5.h\n" - "ldr x10, [x6, #0x90]\n" + "ldr x24, [x14, #0x90]\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" - "ldr x9, [x6, #0x98]\n" + "ldr x23, [x14, #0x98]\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" - "ldr x20, [x6, #0xa0]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.h, p3/M, z0.h, z8.h\n" - "ldr x28, [x6, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.h, p3/M, z0.h, z14.h\n" - "ldr x27, [x6, #0xb0]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" - "ldr x19, [x6, #0xb8]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.h, p3/M, z1.h, z11.h\n" - "ldr x26, [x6, #0xc0]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z5.h\n" - "ldr x25, [x6, #0xc8]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" - "ldr x24, [x6, #0xd0]\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "addvl x5, x5, #16\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n" - "ldr x23, [x6, #0xd8]\n" + "ld1h { z16.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" - "ldr x22, [x6, #0xe0]\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" - "ldr x21, [x6, #0xe8]\n" + "ldr x22, [x14, #0xe0]\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" - "ldr x20, [x6, #0xf0]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.h, p3/M, z0.h, z5.h\n" - "ldr x19, [x6, #0xf8]\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ldr x12, [x6, #0x100]\n" "fmla z29.h, p3/M, z1.h, z6.h\n" - "ldr x11, [x6, #0x108]\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.h, p3/M, z2.h, z9.h\n" - "ldr x10, [x6, #0x110]\n" "fmla z29.h, p3/M, z2.h, z10.h\n" - "ldr x9, [x6, #0x118]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z16.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" - "ldp x12, x11, [x6, #0x0]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" - "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z5.h\n" - "ld1h { z5.h }, p1/Z, [x12, x8, LSL #1]\n" + "ld1h { z5.h }, p1/Z, [x26, x12, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n" - "inch x7\n" + "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n" + "inch x13\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ldp x10, x9, [x6, #0x10]\n" - "whilelt p2.h, x7, %x[n_channels]\n" + "ldp x24, x23, [x14, #0x10]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z31.h, p3/M, z3.h, z6.h\n" - "ld1h { z6.h }, p1/Z, [x11, x8, LSL #1]\n" - "ldp x20, x28, [x6, #0x20]\n" + "ld1h { z6.h }, p1/Z, [x25, x12, LSL #1]\n" + "ldp x22, x21, [x14, #0x20]\n" "fmla z30.h, p3/M, z3.h, z8.h\n" - "ldp x27, x19, [x6, #0x30]\n" + "ldp x20, x19, [x14, #0x30]\n" "fmla z29.h, p3/M, z3.h, z11.h\n" - "ld1h { z7.h }, p1/Z, [x10, x8, LSL #1]\n" + "ld1h { z7.h }, p1/Z, [x24, x12, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z13.h }, p1/Z, [x28, x8, LSL #1]\n" + "ld1h { z13.h }, p1/Z, [x21, x12, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p1/Z, [x9, x8, LSL #1]\n" + "ld1h { z8.h }, p1/Z, [x23, x12, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z11.h }, p1/Z, [x27, x8, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x20, x12, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x19, x8, LSL #1]\n" + "ld1h { z12.h }, p1/Z, [x19, x12, LSL #1]\n" "fmla z28.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x20, x8, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x22, x12, LSL #1]\n" "fmax z31.h, p3/M, z31.h, z18.h\n" - "ldp x26, x25, [x6, #0x40]\n" + "ldp x26, x25, [x14, #0x40]\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" - "addvl x5, x5, #16\n" + "addvl x15, x15, #16\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "ld1h { z10.h }, p1/Z, [x26, x8, LSL #1]\n" - "ld1h { z14.h }, p1/Z, [x25, x8, LSL #1]\n" + "ld1h { z10.h }, p1/Z, [x26, x12, LSL #1]\n" + "ld1h { z14.h }, p1/Z, [x25, x12, LSL #1]\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "inch x8\n" + "inch x12\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias - "cmp x8, %x[n_channels]\n" + "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias - "addvl x5, x5, #-6\n" + "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z31.h }, p0, [x17, x15, LSL #1]\n" - "mov z31.d, z16.d\n" - "st1h { z30.h }, p0, [x16, x15, LSL #1]\n" - "mov z30.d, z16.d\n" - "st1h { z29.h }, p0, [x14, x15, LSL #1]\n" - "mov z29.d, z16.d\n" - "st1h { z28.h }, p0, [x13, x15, LSL #1]\n" - "mov z28.d, z16.d\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ldr x24, [x6, #0x50]\n" - "inch x15\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" - "ldr x23, [x6, #0x58]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ldr x24, [x14, #0x50]\n" + "inch x9\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" + "ldr x23, [x14, #0x58]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" - "ldr x22, [x6, #0x60]\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" + "ldr x22, [x14, #0x60]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x6, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.h, p3/M, z1.h, z8.h\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias - "ldr x20, [x6, #0x70]\n" + "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n" + "ldr x20, [x14, #0x70]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ldr x19, [x6, #0x78]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.h, p3/M, z2.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias - "ldr x12, [x6, #0x80]\n" + "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n" + "ldr x26, [x14, #0x80]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" - "ldr x11, [x6, #0x88]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.h, p3/M, z3.h, z5.h\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias - "ldr x10, [x6, #0x90]\n" + "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias - "ldr x9, [x6, #0x98]\n" + "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" - "ldr x20, [x6, #0xa0]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.h, p3/M, z0.h, z8.h\n" - "ldr x28, [x6, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.h, p3/M, z0.h, z14.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias - "ldr x27, [x6, #0xb0]\n" + "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" - "ldr x19, [x6, #0xb8]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.h, p3/M, z1.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias - "ldr x26, [x6, #0xc0]\n" + "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z5.h\n" - "ldr x25, [x6, #0xc8]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias - "addvl x5, x5, #16\n" + "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n" - "ldr x24, [x6, #0xd0]\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" - "ldr x23, [x6, #0xd8]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias - "ldr x22, [x6, #0xe0]\n" + "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias - "ldr x21, [x6, #0xe8]\n" + "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n" + "ldr x22, [x14, #0xe0]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" - "ldr x20, [x6, #0xf0]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.h, p3/M, z0.h, z5.h\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias - "ldr x19, [x6, #0xf8]\n" + "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ldr x12, [x6, #0x100]\n" "fmla z29.h, p3/M, z1.h, z6.h\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias - "ldr x11, [x6, #0x108]\n" + "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.h, p3/M, z2.h, z9.h\n" - "ldr x10, [x6, #0x110]\n" "fmla z29.h, p3/M, z2.h, z10.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias - "ldr x9, [x6, #0x118]\n" + "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" "fmla z31.h, p3/M, z2.h, z5.h\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" "fmla z31.h, p3/M, z3.h, z6.h\n" "fmla z30.h, p3/M, z3.h, z8.h\n" @@ -540,20 +532,20 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "fmax z29.h, p3/M, z29.h, z18.h\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x17, x15, LSL #1]\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x16, x15, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z29.h }, p0, [x14, x15, LSL #1]\n" - "st1h { z28.h }, p0, [x13, x15, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 05e82d4e76..e07e6314f8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst +class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index eb632eb4fe..571246be3e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,277 +88,229 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( __asm__ __volatile__( "ptrue p3.b\n" - "mov x8, #0x0\n" "mov x17, #0x0\n" + "mov x16, #0x0\n" "1:" // Tile loop - "str x8, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "mov x21, #0x2\n" - "str x17, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "cntb x16\n" - "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" - "add x16, x16, XZR, LSL #4\n" - "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" - "cntb x14\n" - "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" - "cntb x12\n" - "ldr x11, [%x[params_struct], %[offsetof_args_inptr]]\n" - "mul x19, x8, x20\n" // offset = tile_i * ld_input_row - "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" - "madd x19, x17, x13, x19\n" // offset += tile_j * ld_input_col - "ldr x10, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" - "mul x19, x19, x21\n" // offset *= kernel_stride * output_size - "ldr x9, [%x[params_struct], %[offsetof_args_outptr]]\n" - "add x11, x11, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) + "str x17, [%x[params_struct], %[offsetof_args_tile_i]]\n" + "mov x23, #0x2\n" + "str x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "mov x15, #0x2\n" + "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x13, #0x0\n" + "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" + "cntw x12\n" + "ldr x11, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" + "sub x21, XZR, x12\n" + "ldr x10, [%x[params_struct], %[offsetof_args_inptr]]\n" + "mul x19, x17, x22\n" // offset = tile_i * ld_input_row + "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" + "madd x19, x16, x11, x19\n" // offset += tile_j * ld_input_col + "ldr x9, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" + "mul x19, x19, x23\n" // offset *= kernel_stride * output_size + "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n" + "add x10, x10, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "add x28, x11, x20, LSL #2\n" + "add x27, x10, x22, LSL #2\n" "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "add x27, x28, x20, LSL #2\n" - "ld1w { z16.s }, p3/Z, [x15]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" - "add x26, x27, x20, LSL #2\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" - "add x25, x13, x13\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" - "add x24, x25, x13\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" - "add x14, x14, x13, LSL #4\n" - "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" - "add x12, x12, x25, LSL #4\n" - "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" - "cntb x23\n" - "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" - "add x23, x23, x24, LSL #4\n" - "prfm pldl1keep, [x28, x14]\n" - "mov x20, #0x2\n" - "prfm pldl1keep, [x11, x16]\n" - "mul x19, x8, x22\n" // offset = tile_i * ld_output_row - "prfm pldl1keep, [x11, x23]\n" - "madd x19, x17, x10, x19\n" // offset += tile_j * ld_output_col - "prfm pldl1keep, [x28, x12]\n" - "mul x19, x19, x20\n" // offset *= output_tile_size - "prfm pldl1keep, [x27, x14]\n" - "add x9, x9, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "mov x21, #0x0\n" - "add x22, x9, x22, LSL #2\n" - "cntw x20\n" - "sub x19, XZR, x20\n" + "add x26, x27, x22, LSL #2\n" + "ld1w { z16.s }, p3/Z, [x14]\n" + "add x25, x26, x22, LSL #2\n" + "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" + "add x24, x11, x11\n" + "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" + "add x23, x24, x11\n" + "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" + "mul x19, x17, x20\n" // offset = tile_i * ld_output_row + "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" + "madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col + "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" + "mul x19, x19, x15\n" // offset *= output_tile_size + "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" + "add x28, x28, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) + "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" + "add x22, x28, x20, LSL #2\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x11]\n" - "addvl x15, x15, #16\n" - "ld1w { z11.s }, p2/Z, [x11, x24, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" - "addvl x15, x15, #-6\n" - "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x27, x11, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10]\n" + "addvl x14, x14, #16\n" + "ld1w { z11.s }, p2/Z, [x10, x23, LSL #2]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" + "addvl x14, x14, #-6\n" + "ld1w { z12.s }, p2/Z, [x27, x24, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x26, x11, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x16]\n" - "whilelt p1.s, x20, %x[n_channels]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x23]\n" - "incw x19\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x14]\n" - "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" "incw x21\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "mov p0.b, p2.b\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x25]\n" + "incw x13\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x11, x12]\n" - "incw x20\n" + "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n" + "incw x12\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x27, x12]\n" + "ld1w { z16.s }, p3/Z, [x14]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x28, x16]\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x28, x23]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n" + "addvl x10, x10, #1\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "addvl x11, x11, #1\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x16]\n" - "prfm pldl1keep, [x27, x23]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" - "prfm pldl1keep, [x26, x14]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x26, x12]\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28]\n" + "ld1w { z11.s }, p2/Z, [x27]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z16.s }, p3/Z, [x15]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n" + "addvl x27, x27, #1\n" "fmla z29.s, p3/M, z5.s, z10.s\n" - "addvl x28, x28, #1\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x28, x14]\n" - "prfm pldl1keep, [x11, x16]\n" + "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x11, x23]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x27]\n" + "ld1w { z9.s }, p2/Z, [x26]\n" + "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "prfm pldl1keep, [x28, x12]\n" + "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" "fmla z28.s, p3/M, z2.s, z12.s\n" - "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" "fmla z31.s, p3/M, z8.s, z10.s\n" - "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n" + "addvl x26, x26, #1\n" "fmla z29.s, p3/M, z3.s, z9.s\n" - "addvl x27, x27, #1\n" + "ld1w { z13.s }, p1/Z, [x26, x11, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" - "ld1w { z13.s }, p1/Z, [x27, x13, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n" - "whilelt p2.s, x21, %x[n_channels]\n" + "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z29.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x27, x14]\n" - "addvl x26, x26, #1\n" + "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" + "addvl x25, x25, #1\n" "fmla z31.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p1/Z, [x28, x13, LSL #2]\n" - "cmp x20, %x[n_channels]\n" + "ld1w { z9.s }, p1/Z, [x27, x11, LSL #2]\n" + "cmp x12, %x[n_channels]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p1/Z, [x11]\n" + "ld1w { z10.s }, p1/Z, [x10]\n" "fmla z28.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x11, x24, LSL #2]\n" - "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z11.s }, p1/Z, [x10, x23, LSL #2]\n" + "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "addvl x14, x14, #16\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" + "ld1w { z12.s }, p1/Z, [x27, x24, LSL #2]\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x28, x25, LSL #2]\n" - "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" + "addvl x14, x14, #-6\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "addvl x15, x15, #16\n" + "st1w { z31.s }, p0, [x28]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" - "addvl x15, x15, #-6\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z31.s }, p0, [x9]\n" - "mov z31.d, z16.d\n" + "st1w { z30.s }, p0, [x28, x9, LSL #2]\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" + "addvl x28, x28, #1\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z30.s }, p0, [x9, x10, LSL #2]\n" - "addvl x9, x9, #1\n" - "mov z30.d, z16.d\n" "st1w { z29.s }, p0, [x22]\n" - "mov z29.d, z16.d\n" - "st1w { z28.s }, p0, [x22, x10, LSL #2]\n" - "mov z28.d, z16.d\n" + "st1w { z28.s }, p0, [x22, x9, LSL #2]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x16]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x23]\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x14]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26]\n" - "prfm pldl1keep, [x11, x12]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" + "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "add x21, x17, #0x1\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x25]\n" + "add x16, x16, #0x1\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n" + "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x27, x12]\n" - "prfm pldl1keep, [x28, x16]\n" + "cmp x16, x19\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x28, x23]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" + "csel x16, x16, XZR, LT\n" + "fmla z28.s, p3/M, z1.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n" + "csel x17, x17, x21, LT\n" "fmla z29.s, p3/M, z6.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n" - "prfm pldl1keep, [x27, x16]\n" + "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n" + "cmp x17, x20\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x23]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" + "fmla z28.s, p3/M, z3.s, z13.s\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x26, x14]\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28]\n" - "prfm pldl1keep, [x26, x12]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ldr x8, [%x[params_struct], %[offsetof_args_tile_i]]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "add x21, x8, #0x1\n" + "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x27]\n" "fmla z29.s, p3/M, z5.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "ldr x17, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "add x17, x17, #0x1\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x27]\n" - "fmla z31.s, p3/M, z8.s, z10.s\n" - "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "ld1w { z9.s }, p2/Z, [x26]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "cmp x17, x19\n" - "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "csel x17, x17, XZR, LT\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" - "csel x8, x8, x21, LT\n" + "fmla z31.s, p3/M, z8.s, z10.s\n" + "fmla z30.s, p3/M, z7.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z12.s\n" - "cmp x8, x20\n" - "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n" - "fmla z31.s, p3/M, z6.s, z9.s\n" "fmla z29.s, p3/M, z3.s, z9.s\n" + "fmla z31.s, p3/M, z3.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n" + "fmla z30.s, p3/M, z5.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" - "fmla z30.s, p3/M, z8.s, z10.s\n" "fmla z29.s, p3/M, z7.s, z11.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" + "fmla z31.s, p3/M, z6.s, z9.s\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" "fmla z28.s, p3/M, z6.s, z11.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x9]\n" + "fmax z31.s, p3/M, z31.s, z18.s\n" + "fmax z30.s, p3/M, z30.s, z18.s\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z30.s }, p0, [x9, x10, LSL #2]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" + "fmin z31.s, p3/M, z31.s, z17.s\n" + "st1w { z31.s }, p0, [x28]\n" + "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" + "st1w { z30.s }, p0, [x28, x9, LSL #2]\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" "st1w { z29.s }, p0, [x22]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x22, x10, LSL #2]\n" + "st1w { z28.s }, p0, [x22, x9, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index fb41ca0754..77a6c683b0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -54,22 +54,22 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( const float max ) : outptrs(outptrs), params(params), min(min), max(max) { - inptrs[0] = input_ptrs[0]; - inptrs[1] = input_ptrs[1]; - inptrs[2] = input_ptrs[2]; - inptrs[3] = input_ptrs[3]; - inptrs[4] = input_ptrs[4]; - inptrs[5] = input_ptrs[5]; - inptrs[6] = input_ptrs[6]; - inptrs[7] = input_ptrs[7]; - inptrs[8] = input_ptrs[8]; - inptrs[9] = input_ptrs[9]; - inptrs[10] = input_ptrs[10]; - inptrs[11] = input_ptrs[11]; - inptrs[12] = input_ptrs[12]; - inptrs[13] = input_ptrs[13]; - inptrs[14] = input_ptrs[14]; - inptrs[15] = input_ptrs[15]; + inptrs[0] = input_ptrs[5]; + inptrs[1] = input_ptrs[0]; + inptrs[2] = input_ptrs[3]; + inptrs[3] = input_ptrs[6]; + inptrs[4] = input_ptrs[9]; + inptrs[5] = input_ptrs[12]; + inptrs[6] = input_ptrs[15]; + inptrs[7] = input_ptrs[1]; + inptrs[8] = input_ptrs[2]; + inptrs[9] = input_ptrs[10]; + inptrs[10] = input_ptrs[4]; + inptrs[11] = input_ptrs[7]; + inptrs[12] = input_ptrs[8]; + inptrs[13] = input_ptrs[11]; + inptrs[14] = input_ptrs[13]; + inptrs[15] = input_ptrs[14]; } }; @@ -78,216 +78,194 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n" - "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x4, ALL, MUL #2\n" - "ldp x5, x6, [x19, #0x0]\n" - "mov x7, #0x0\n" - "ldp x8, x17, [x19, #0x10]\n" - "cntw x16\n" - "ldp x15, x14, [x19, #0x20]\n" - "sub x13, XZR, x16\n" - "ldp x12, x11, [x19, #0x30]\n" - "whilelt p2.s, XZR, %x[n_channels]\n" - "ldp x10, x9, [x19, #0x40]\n" - "cmp x16, %x[n_channels]\n" - "ldp x28, x27, [x19, #0x50]\n" - "ldp x26, x25, [x19, #0x60]\n" - "ldp x24, x23, [x19, #0x70]\n" - "ldp x22, x21, [x2, #0x0]\n" - "ldp x20, x19, [x2, #0x10]\n" + "mov x13, #0x0\n" "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1w { z16.s }, p3/Z, [x3]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n" - "addvl x3, x3, #16\n" - "ld1w { z9.s }, p2/Z, [x14, x7, LSL #2]\n" - "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n" - "addvl x3, x3, #-6\n" - "prfm pldl1keep, [x14, x4]\n" - "ld1w { z10.s }, p2/Z, [x5, x7, LSL #2]\n" - "prfm pldl1keep, [x5, x4]\n" - "ld1w { z11.s }, p2/Z, [x17, x7, LSL #2]\n" - "prfm pldl1keep, [x17, x4]\n" - "ld1w { z12.s }, p2/Z, [x12, x7, LSL #2]\n" - "prfm pldl1keep, [x12, x4]\n" - "ld1w { z13.s }, p2/Z, [x9, x7, LSL #2]\n" - "prfm pldl1keep, [x9, x4]\n" + "cntw x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" + "whilelt p2.s, XZR, %x[n_channels]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldr x22, [x14, #0x20]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x4]\n" - "whilelt p1.s, x16, %x[n_channels]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x23, x4]\n" - "incw x13\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x6, x4]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "ldr x21, [x14, #0x28]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x14, #0x30]\n" + "incw x9\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "ldr x19, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n" - "prfm pldl1keep, [x8, x4]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x28, x4]\n" + "ldr x24, [x14, #0x50]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x15, x4]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x11, x4]\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x10, x4]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x4]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" - "prfm pldl1keep, [x25, x4]\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x24, x4]\n" + "ldr x19, [x14, #0x78]\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "addvl x4, x4, #1\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n" - "prfm pldl1keep, [x14, x4]\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x5, x4]\n" + "ldp x24, x23, [x14, #0x10]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x17, x4]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x12, x4]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z13.s }, p1/Z, [x9, x16, LSL #2]\n" + "ldr x22, [x14, #0x20]\n" "fmla z28.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x9, x4]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z8.s, z10.s\n" - "ld1w { z16.s }, p3/Z, [x3]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" - "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n" + "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" - "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n" - "incw x7\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "incw x13\n" "fmla z29.s, p3/M, z7.s, z11.s\n" - "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n" - "whilelt p2.s, x7, %x[n_channels]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z31.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n" "fmla z28.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x17, x16, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p1/Z, [x5, x16, LSL #2]\n" - "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n" + "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" + "addvl x15, x15, #16\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x12, x16, LSL #2]\n" - "incw x16\n" + "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n" + "incw x12\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n" - "cmp x16, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n" - "addvl x3, x3, #16\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n" - "addvl x3, x3, #-6\n" + "fmin z31.s, p3/M, z31.s, z17.s\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z31.s }, p0, [x22, x13, LSL #2]\n" - "mov z31.d, z16.d\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x21, x13, LSL #2]\n" - "mov z30.d, z16.d\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z29.s }, p0, [x20, x13, LSL #2]\n" - "mov z29.d, z16.d\n" - "st1w { z28.s }, p0, [x19, x13, LSL #2]\n" - "mov z28.d, z16.d\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x4]\n" - "incw x13\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x23, x4]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "ldr x21, [x14, #0x28]\n" + "incw x9\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x6, x4]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n" - "prfm pldl1keep, [x8, x4]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "ldr x19, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x28, x4]\n" - "prfm pldl1keep, [x15, x4]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x24, [x14, #0x50]\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x11, x4]\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z9.s\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n" - "prfm pldl1keep, [x10, x4]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x4]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" - "prfm pldl1keep, [x25, x4]\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n" - "prfm pldl1keep, [x24, x4]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x19, [x14, #0x78]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z10.s\n" "fmla z28.s, p3/M, z4.s, z10.s\n" "fmla z31.s, p3/M, z2.s, z9.s\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" "fmla z28.s, p3/M, z2.s, z12.s\n" "fmla z31.s, p3/M, z8.s, z10.s\n" "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z11.s\n" "fmla z31.s, p3/M, z6.s, z9.s\n" "fmla z28.s, p3/M, z6.s, z11.s\n" @@ -298,21 +276,21 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "fmax z30.s, p3/M, z30.s, z18.s\n" "fmax z29.s, p3/M, z29.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x22, x13, LSL #2]\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x21, x13, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "st1w { z29.s }, p0, [x20, x13, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x19, x13, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index ef5f4187f9..eb9de9fa01 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst +class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 6bc333be41..c485b7dde0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,167 +88,123 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( __asm__ __volatile__( "ptrue p3.b\n" - "mov x3, #0x0\n" - "mov x4, #0x0\n" + "mov x6, #0x0\n" + "mov x7, #0x0\n" "1:" // Tile loop - "str x3, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "mov x22, #0x3\n" - "str x4, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "cntb x5\n" - "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n" - "add x5, x5, XZR, LSL #4\n" - "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" - "cntb x7\n" - "ldr x8, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" - "cntb x17\n" - "ldr x16, [%x[params_struct], %[offsetof_args_inptr]]\n" - "mul x19, x3, x20\n" // offset = tile_i * ld_input_row - "ldr x21, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" - "madd x19, x4, x8, x19\n" // offset += tile_j * ld_input_col - "ldr x15, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" - "mul x19, x19, x22\n" // offset *= kernel_stride * output_size - "ldr x14, [%x[params_struct], %[offsetof_args_outptr]]\n" - "add x16, x16, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) + "str x6, [%x[params_struct], %[offsetof_args_tile_i]]\n" + "mov x24, #0x3\n" + "str x7, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "mov x23, #0x3\n" + "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x17, #0x0\n" + "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" + "cntw x16\n" + "ldr x15, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" + "sub x21, XZR, x16\n" + "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n" + "mul x19, x6, x22\n" // offset = tile_i * ld_input_row + "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" + "madd x19, x7, x15, x19\n" // offset += tile_j * ld_input_col + "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" + "mul x19, x19, x24\n" // offset *= kernel_stride * output_size + "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n" + "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "add x13, x16, x20, LSL #2\n" + "add x11, x14, x22, LSL #2\n" "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "add x12, x13, x20, LSL #2\n" - "ld1w { z16.s }, p3/Z, [x6]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" - "add x11, x12, x20, LSL #2\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" - "add x10, x11, x20, LSL #2\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" - "add x9, x8, x8\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" - "add x28, x9, x8\n" - "mov z27.d, z16.d\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" - "add x27, x28, x8\n" - "mov z26.d, z16.d\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" - "add x7, x7, x8, LSL #4\n" - "mov z25.d, z16.d\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" - "add x17, x17, x9, LSL #4\n" - "mov z24.d, z16.d\n" - "prfm pldl1keep, [x12, x17]\n" - "cntb x26\n" - "mov z23.d, z16.d\n" - "prfm pldl1keep, [x16, x5]\n" - "add x26, x26, x28, LSL #4\n" - "cntb x25\n" - "mov x20, #0x3\n" - "add x25, x25, x27, LSL #4\n" - "prfm pldl1keep, [x16, x25]\n" - "prfm pldl1keep, [x10, x5]\n" - "mul x19, x3, x21\n" // offset = tile_i * ld_output_row - "prfm pldl1keep, [x13, x17]\n" - "madd x19, x4, x15, x19\n" // offset += tile_j * ld_output_col - "add x24, x15, x15\n" - "mul x19, x19, x20\n" // offset *= output_tile_size - "add x14, x14, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "add x23, x14, x21, LSL #2\n" - "add x22, x23, x21, LSL #2\n" - "mov x21, #0x0\n" - "cntw x20\n" - "sub x19, XZR, x20\n" + "add x10, x11, x22, LSL #2\n" + "ld1w { z16.s }, p3/Z, [x8]\n" + "add x9, x10, x22, LSL #2\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" + "add x28, x9, x22, LSL #2\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" + "add x27, x15, x15\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" + "add x26, x27, x15\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "add x25, x26, x15\n" + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "mul x19, x6, x20\n" // offset = tile_i * ld_output_row + "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" + "madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col + "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" + "mul x19, x19, x23\n" // offset *= output_tile_size + "add x24, x13, x13\n" + "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) + "add x23, x12, x20, LSL #2\n" + "add x22, x23, x20, LSL #2\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z9.s }, p2/Z, [x12, x9, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x16]\n" - "addvl x6, x6, #16\n" - "ld1w { z11.s }, p2/Z, [x16, x27, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" - "ld1w { z12.s }, p2/Z, [x10]\n" - "ld1w { z13.s }, p2/Z, [x13, x9, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x14]\n" + "addvl x8, x8, #16\n" + "ld1w { z11.s }, p2/Z, [x14, x25, LSL #2]\n" + "cmp x16, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" + "addvl x8, x8, #-6\n" + "ld1w { z12.s }, p2/Z, [x28]\n" + "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x10, x25]\n" - "whilelt p1.s, x20, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "prfm pldl1keep, [x12, x7]\n" - "incw x19\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x16, x7]\n" - "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x16, x26]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "whilelt p1.s, x16, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" "incw x21\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x12, x26]\n" - "incw x20\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x13, x5]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x13, x25]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x5]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x11, x17]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "mov p0.b, p2.b\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "incw x17\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "incw x16\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "ld1w { z16.s }, p3/Z, [x8]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x11, x25]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x10, x7]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x13, x7]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "prfm pldl1keep, [x13, x26]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x26]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x7]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x16, x17]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x11, x26]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x12, x5]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x12, x25]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13]\n" + "ld1w { z11.s }, p2/Z, [x11]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x17]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z16.s }, p3/Z, [x6]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" "fmla z26.s, p3/M, z4.s, z10.s\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11]\n" + "ld1w { z12.s }, p2/Z, [x9]\n" "fmla z29.s, p3/M, z7.s, z10.s\n" "fmla z24.s, p3/M, z2.s, z10.s\n" "fmla z23.s, p3/M, z1.s, z10.s\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" "fmla z26.s, p3/M, z6.s, z10.s\n" "fmla z25.s, p3/M, z5.s, z10.s\n" @@ -258,198 +214,164 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.s, p3/M, z8.s, z11.s\n" "fmla z25.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n" - "addvl x13, x13, #1\n" + "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" + "addvl x11, x11, #1\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "fmla z30.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" "fmla z30.s, p3/M, z5.s, z11.s\n" "fmla z26.s, p3/M, z1.s, z11.s\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n" - "addvl x16, x16, #1\n" + "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n" + "addvl x14, x14, #1\n" "fmla z24.s, p3/M, z8.s, z13.s\n" - "ld1w { z10.s }, p1/Z, [x16]\n" + "ld1w { z10.s }, p1/Z, [x14]\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n" - "addvl x11, x11, #1\n" + "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" + "addvl x9, x9, #1\n" "fmla z28.s, p3/M, z7.s, z12.s\n" "fmla z27.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12]\n" + "ld1w { z12.s }, p2/Z, [x10]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" "fmla z30.s, p3/M, z1.s, z11.s\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n" - "addvl x12, x12, #1\n" + "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" + "addvl x10, x10, #1\n" "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z9.s }, p1/Z, [x12, x9, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n" "fmla z26.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x12, x17]\n" "fmla z24.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x16, x5]\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n" - "whilelt p2.s, x21, %x[n_channels]\n" + "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n" + "whilelt p2.s, x17, %x[n_channels]\n" "fmla z31.s, p3/M, z6.s, z12.s\n" - "prfm pldl1keep, [x16, x25]\n" - "addvl x10, x10, #1\n" + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "addvl x28, x28, #1\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x10, x5]\n" - "cmp x20, %x[n_channels]\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "cmp x16, %x[n_channels]\n" "fmla z25.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x10]\n" + "ld1w { z12.s }, p1/Z, [x28]\n" "fmla z29.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x13, x17]\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" "fmla z26.s, p3/M, z5.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" "fmla z23.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x16, x27, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x14, x25, LSL #2]\n" "fmla z24.s, p3/M, z7.s, z13.s\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" "fmla z25.s, p3/M, z8.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" "fmla z23.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p1/Z, [x13, x9, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x11, x27, LSL #2]\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" - "addvl x6, x6, #16\n" + "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" + "addvl x8, x8, #16\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" + "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" + "fmax z29.s, p3/M, z29.s, z18.s\n" + "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" + "addvl x8, x8, #-6\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z31.s }, p0, [x14]\n" - "mov z31.d, z16.d\n" + "st1w { z31.s }, p0, [x12]\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x14, x15, LSL #2]\n" - "mov z30.d, z16.d\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z29.s }, p0, [x14, x24, LSL #2]\n" - "mov z29.d, z16.d\n" - "addvl x14, x14, #1\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" + "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" "fmax z27.s, p3/M, z27.s, z18.s\n" - "st1w { z28.s }, p0, [x23]\n" - "mov z28.d, z16.d\n" "fmax z26.s, p3/M, z26.s, z18.s\n" + "st1w { z29.s }, p0, [x12, x24, LSL #2]\n" + "fmin z28.s, p3/M, z28.s, z17.s\n" + "addvl x12, x12, #1\n" "fmax z25.s, p3/M, z25.s, z18.s\n" - "fmax z24.s, p3/M, z24.s, z18.s\n" + "st1w { z28.s }, p0, [x23]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" - "st1w { z27.s }, p0, [x23, x15, LSL #2]\n" - "mov z27.d, z16.d\n" "fmin z26.s, p3/M, z26.s, z17.s\n" + "st1w { z27.s }, p0, [x23, x13, LSL #2]\n" + "fmin z25.s, p3/M, z25.s, z17.s\n" + "fmax z24.s, p3/M, z24.s, z18.s\n" "st1w { z26.s }, p0, [x23, x24, LSL #2]\n" - "mov z26.d, z16.d\n" "addvl x23, x23, #1\n" - "fmin z25.s, p3/M, z25.s, z17.s\n" + "fmax z23.s, p3/M, z23.s, z18.s\n" "st1w { z25.s }, p0, [x22]\n" - "mov z25.d, z16.d\n" "fmin z24.s, p3/M, z24.s, z17.s\n" - "st1w { z24.s }, p0, [x22, x15, LSL #2]\n" - "mov z24.d, z16.d\n" - "fmax z23.s, p3/M, z23.s, z18.s\n" "fmin z23.s, p3/M, z23.s, z17.s\n" + "st1w { z24.s }, p0, [x22, x13, LSL #2]\n" "st1w { z23.s }, p0, [x22, x24, LSL #2]\n" - "mov z23.d, z16.d\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x10, x25]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "prfm pldl1keep, [x12, x7]\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x16, x7]\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x16, x26]\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x12, x26]\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x13, x5]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x13, x25]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x5]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x11, x17]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "add x21, x6, #0x1\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" + "add x7, x7, #0x1\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "cmp x7, x19\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "csel x7, x7, XZR, LT\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "csel x6, x6, x21, LT\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "cmp x6, x20\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x11, x25]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x10, x7]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x13, x7]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "prfm pldl1keep, [x13, x26]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x26]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x7]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x16, x17]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x11, x26]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x12, x5]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x12, x25]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13]\n" + "ld1w { z11.s }, p2/Z, [x11]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x17]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ldr x3, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "add x21, x3, #0x1\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11]\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ldr x4, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "add x4, x4, #0x1\n" - "fmla z30.s, p3/M, z8.s, z10.s\n" - "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" - "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "cmp x4, x19\n" "fmla z26.s, p3/M, z4.s, z10.s\n" + "fmla z30.s, p3/M, z2.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x9]\n" + "fmla z29.s, p3/M, z7.s, z10.s\n" "fmla z24.s, p3/M, z2.s, z10.s\n" - "csel x4, x4, XZR, LT\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n" - "csel x3, x3, x21, LT\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "cmp x3, x20\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" "fmla z26.s, p3/M, z6.s, z10.s\n" "fmla z25.s, p3/M, z5.s, z10.s\n" @@ -459,36 +381,36 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.s, p3/M, z8.s, z11.s\n" "fmla z25.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "fmla z30.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" "fmla z30.s, p3/M, z5.s, z11.s\n" "fmla z26.s, p3/M, z1.s, z11.s\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z13.s\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" "fmla z27.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12]\n" + "ld1w { z12.s }, p2/Z, [x10]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" "fmla z30.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z13.s\n" "fmla z26.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z5.s, z13.s\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z12.s\n" "fmla z28.s, p3/M, z3.s, z12.s\n" "fmla z25.s, p3/M, z0.s, z12.s\n" @@ -502,13 +424,13 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmax z30.s, p3/M, z30.s, z18.s\n" "fmax z29.s, p3/M, z29.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x14]\n" + "st1w { z31.s }, p0, [x12]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x14, x15, LSL #2]\n" + "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z18.s\n" "fmax z27.s, p3/M, z27.s, z18.s\n" - "st1w { z29.s }, p0, [x14, x24, LSL #2]\n" + "st1w { z29.s }, p0, [x12, x24, LSL #2]\n" "fmax z26.s, p3/M, z26.s, z18.s\n" "fmax z25.s, p3/M, z25.s, z18.s\n" "fmax z24.s, p3/M, z24.s, z18.s\n" @@ -516,23 +438,23 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "st1w { z28.s }, p0, [x23]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" "fmin z26.s, p3/M, z26.s, z17.s\n" - "st1w { z27.s }, p0, [x23, x15, LSL #2]\n" + "st1w { z27.s }, p0, [x23, x13, LSL #2]\n" "fmin z25.s, p3/M, z25.s, z17.s\n" "fmin z24.s, p3/M, z24.s, z17.s\n" "st1w { z26.s }, p0, [x23, x24, LSL #2]\n" "fmax z23.s, p3/M, z23.s, z18.s\n" "st1w { z25.s }, p0, [x22]\n" "fmin z23.s, p3/M, z23.s, z17.s\n" - "st1w { z24.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z24.s }, p0, [x22, x13, LSL #2]\n" "st1w { z23.s }, p0, [x22, x24, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 3877ae2f03..72b182679d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -87,413 +87,343 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x17, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" - "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x14, ALL, MUL #2\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "mov x13, #0x0\n" - "ld1w { z16.s }, p3/Z, [x16]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "cntw x12\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" "sub x11, XZR, x12\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "cmp x12, %x[n_channels]\n" - "mov z27.d, z16.d\n" - "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" - "mov z26.d, z16.d\n" - "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" - "mov z25.d, z16.d\n" - "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" - "addvl x16, x16, #16\n" - "mov z24.d, z16.d\n" - "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" - "mov z23.d, z16.d\n" - "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" - "addvl x16, x16, #-6\n" - "ldp x10, x22, [x15, #0x0]\n" - "ldp x9, x28, [x15, #0x10]\n" - "ldr x24, [x15, #0x20]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n" - "prfm pldl1keep, [x10, x14]\n" - "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n" - "prfm pldl1keep, [x22, x14]\n" - "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" - "prfm pldl1keep, [x9, x14]\n" - "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" - "prfm pldl1keep, [x28, x14]\n" - "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n" - "prfm pldl1keep, [x24, x14]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ldr x26, [x14, #0x20]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x15, #0x28]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" "whilelt p1.s, x12, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x23, [x15, #0x30]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" "incw x11\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x26, [x15, #0x38]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x27, x14]\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x23, x14]\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x14]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "ldr x25, [x15, #0x40]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "ldr x19, [x15, #0x48]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "ldr x24, [x15, #0x50]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x24, x14]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "ldr x23, [x15, #0x58]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x22, [x15, #0x60]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x23, x14]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x15, #0x68]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x15, #0x70]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x15, #0x78]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z26.s, p3/M, z4.s, z10.s\n" - "ldr x10, [x15, #0x80]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z10.s\n" - "ldr x22, [x15, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x10, x14]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x9, [x15, #0x90]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x9, x14]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" - "ldr x28, [x15, #0x98]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" - "ldr x24, [x15, #0xa0]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x28, x14]\n" "fmla z25.s, p3/M, z5.s, z10.s\n" - "ldr x27, [x15, #0xa8]\n" "fmla z28.s, p3/M, z8.s, z10.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z24.s, p3/M, z4.s, z10.s\n" - "ldr x23, [x15, #0xb0]\n" "fmla z23.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x27, x14]\n" "fmla z26.s, p3/M, z8.s, z11.s\n" - "ldr x26, [x15, #0xb8]\n" "fmla z25.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "ldr x25, [x15, #0xc0]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ldp x10, x22, [x15, #0x0]\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x25, x14]\n" - "addvl x14, x14, #1\n" "fmla z30.s, p3/M, z5.s, z11.s\n" - "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n" "fmla z26.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x10, x14]\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z13.s\n" - "ld1w { z10.s }, p1/Z, [x22, x12, LSL #2]\n" + "ldr x26, [x14, #0x20]\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x22, x14]\n" "fmla z27.s, p3/M, z6.s, z12.s\n" - "ldp x9, x28, [x15, #0x10]\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "ldr x24, [x15, #0x20]\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x9, x14]\n" "fmla z30.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x28, x14]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z13.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z26.s, p3/M, z7.s, z13.s\n" - "ldr x22, [x17, #0x0]\n" "fmla z24.s, p3/M, z5.s, z13.s\n" - "ldr x21, [x17, #0x8]\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n" "incw x13\n" "fmla z31.s, p3/M, z6.s, z12.s\n" - "ldr x20, [x17, #0x10]\n" + "ldp x10, x9, [x14, #0x0]\n" "whilelt p2.s, x13, %x[n_channels]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ldr x19, [x17, #0x18]\n" + "ldp x28, x27, [x14, #0x10]\n" "fmla z25.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x28, x12, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z16.s }, p3/Z, [x16]\n" + "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n" "fmla z26.s, p3/M, z5.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n" "fmla z23.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x9, x12, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n" "fmla z25.s, p3/M, z8.s, z13.s\n" - "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n" "fmla z24.s, p3/M, z7.s, z13.s\n" - "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z23.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p1/Z, [x24, x12, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x26, x12, LSL #2]\n" "incw x12\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" "cmp x12, %x[n_channels]\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" - "addvl x16, x16, #16\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" "fmax z27.s, p3/M, z27.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" - "addvl x16, x16, #-6\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" - "mov z31.d, z16.d\n" - "ldr x22, [x17, #0x20]\n" "fmin z29.s, p3/M, z29.s, z17.s\n" + "ldr x22, [x16, #0x20]\n" + "fmax z26.s, p3/M, z26.s, z18.s\n" "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" - "mov z30.d, z16.d\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" - "mov z29.d, z16.d\n" - "ldr x21, [x17, #0x28]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" - "ldr x20, [x17, #0x30]\n" - "fmax z26.s, p3/M, z26.s, z18.s\n" - "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" - "mov z28.d, z16.d\n" - "ldr x19, [x17, #0x38]\n" + "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" + "fmin z26.s, p3/M, z26.s, z17.s\n" + "ldr x21, [x16, #0x28]\n" "fmax z25.s, p3/M, z25.s, z18.s\n" + "ldr x20, [x16, #0x30]\n" + "fmax z24.s, p3/M, z24.s, z18.s\n" + "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" + "fmax z23.s, p3/M, z23.s, z18.s\n" "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" - "mov z27.d, z16.d\n" - "ldr x22, [x17, #0x40]\n" - "fmin z26.s, p3/M, z26.s, z17.s\n" "st1w { z26.s }, p0, [x21, x11, LSL #2]\n" - "mov z26.d, z16.d\n" "fmin z25.s, p3/M, z25.s, z17.s\n" - "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" - "mov z25.d, z16.d\n" - "fmax z24.s, p3/M, z24.s, z18.s\n" - "fmax z23.s, p3/M, z23.s, z18.s\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.s, p3/M, z24.s, z17.s\n" - "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" - "mov z24.d, z16.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.s, p3/M, z23.s, z17.s\n" + "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" + "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" - "mov z23.d, z16.d\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x15, #0x28]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" "incw x11\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x23, [x15, #0x30]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x26, [x15, #0x38]\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x27, x14]\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x23, x14]\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x14]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "ldr x25, [x15, #0x40]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "ldr x19, [x15, #0x48]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "ldr x24, [x15, #0x50]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x24, x14]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "ldr x23, [x15, #0x58]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x22, [x15, #0x60]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x23, x14]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x15, #0x68]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x15, #0x70]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x15, #0x78]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z26.s, p3/M, z4.s, z10.s\n" - "ldr x10, [x15, #0x80]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z10.s\n" - "ldr x22, [x15, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x10, x14]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x9, [x15, #0x90]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x9, x14]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" - "ldr x28, [x15, #0x98]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" - "ldr x24, [x15, #0xa0]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x28, x14]\n" "fmla z25.s, p3/M, z5.s, z10.s\n" - "ldr x27, [x15, #0xa8]\n" "fmla z28.s, p3/M, z8.s, z10.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z24.s, p3/M, z4.s, z10.s\n" - "ldr x23, [x15, #0xb0]\n" "fmla z23.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x27, x14]\n" "fmla z26.s, p3/M, z8.s, z11.s\n" - "ldr x26, [x15, #0xb8]\n" "fmla z25.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "ldr x25, [x15, #0xc0]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ldr x22, [x17, #0x0]\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x25, x14]\n" "fmla z30.s, p3/M, z5.s, z11.s\n" - "ldr x21, [x17, #0x8]\n" "fmla z26.s, p3/M, z1.s, z11.s\n" - "ldr x20, [x17, #0x10]\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z13.s\n" - "ldr x19, [x17, #0x18]\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" "fmla z27.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" "fmla z30.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z13.s\n" "fmla z26.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z5.s, z13.s\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z12.s\n" "fmla z28.s, p3/M, z3.s, z12.s\n" "fmla z25.s, p3/M, z0.s, z12.s\n" @@ -510,21 +440,21 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "ldr x22, [x17, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmax z28.s, p3/M, z28.s, z18.s\n" "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" "fmax z27.s, p3/M, z27.s, z18.s\n" "fmax z26.s, p3/M, z26.s, z18.s\n" "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "ldr x21, [x17, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z25.s, p3/M, z25.s, z18.s\n" - "ldr x20, [x17, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmax z24.s, p3/M, z24.s, z18.s\n" "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" "fmin z26.s, p3/M, z26.s, z17.s\n" - "ldr x19, [x17, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z25.s, p3/M, z25.s, z17.s\n" "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" "fmin z24.s, p3/M, z24.s, z17.s\n" @@ -533,15 +463,15 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" "fmin z23.s, p3/M, z23.s, z17.s\n" "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" - "ldr x22, [x17, #0x40]\n" + "ldr x22, [x16, #0x40]\n" "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index fc9588cd58..d7be9b122e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst +class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 7df8e481c0..84b4b3b72b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -114,45 +114,29 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x14, x15, x22, LSL #2\n" "ld1w { z13.s }, p3/Z, [x4]\n" - "mov z31.d, z13.d\n" - "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n" "add x13, x14, x22, LSL #2\n" - "mov z30.d, z13.d\n" - "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n" "add x12, x13, x22, LSL #2\n" - "mov z29.d, z13.d\n" - "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n" "add x11, x12, x22, LSL #2\n" - "mov z28.d, z13.d\n" - "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n" "add x10, x7, x7\n" - "mov z27.d, z13.d\n" - "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n" "add x9, x10, x7\n" - "mov z26.d, z13.d\n" - "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n" "add x28, x9, x7\n" - "mov z25.d, z13.d\n" - "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n" "add x27, x28, x7\n" - "mov z24.d, z13.d\n" + "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n" "mul x19, x2, x20\n" // offset = tile_i * ld_output_row - "mov z23.d, z13.d\n" + "add x26, x17, x17\n" "madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col - "mov z22.d, z13.d\n" "mul x19, x19, x23\n" // offset *= output_tile_size - "mov z21.d, z13.d\n" "add x16, x16, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "mov z20.d, z13.d\n" - "add x26, x16, x20, LSL #2\n" - "mov z19.d, z13.d\n" - "add x25, x26, x20, LSL #2\n" - "mov z18.d, z13.d\n" + "add x25, x16, x20, LSL #2\n" "add x24, x25, x20, LSL #2\n" - "mov z17.d, z13.d\n" - "add x23, x17, x17\n" - "mov z16.d, z13.d\n" - "add x22, x23, x17\n" + "add x23, x24, x20, LSL #2\n" + "add x22, x26, x17\n" "whilelt p2.s, XZR, %x[n_channels]\n" "ld1w { z9.s }, p2/Z, [x14, x10, LSL #2]\n" "ld1w { z10.s }, p2/Z, [x8]\n" @@ -165,39 +149,38 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ld1w { z12.s }, p2/Z, [x14, x9, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ld1w { z13.s }, p3/Z, [x4]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" "whilelt p1.s, x6, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" "incw x21\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" "mov p0.b, p2.b\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" "incw x5\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" "incw x6\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x11]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" "fmla z29.s, p3/M, z7.s, z12.s\n" "fmla z26.s, p3/M, z5.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" "fmla z22.s, p3/M, z2.s, z12.s\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" "fmla z26.s, p3/M, z7.s, z9.s\n" @@ -206,10 +189,11 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.s, p3/M, z4.s, z9.s\n" "fmla z21.s, p3/M, z3.s, z9.s\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" + "ld1w { z13.s }, p3/Z, [x4]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x15, x27, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" @@ -375,109 +359,93 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "addvl x4, x4, #-6\n" "fmin z31.s, p3/M, z31.s, z14.s\n" "st1w { z31.s }, p0, [x16]\n" - "mov z31.d, z13.d\n" "fmin z30.s, p3/M, z30.s, z14.s\n" - "st1w { z30.s }, p0, [x16, x17, LSL #2]\n" - "mov z30.d, z13.d\n" "fmin z29.s, p3/M, z29.s, z14.s\n" - "st1w { z29.s }, p0, [x16, x23, LSL #2]\n" - "mov z29.d, z13.d\n" + "st1w { z30.s }, p0, [x16, x17, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z15.s\n" "fmax z27.s, p3/M, z27.s, z15.s\n" + "st1w { z29.s }, p0, [x16, x26, LSL #2]\n" "fmax z26.s, p3/M, z26.s, z15.s\n" "fmax z25.s, p3/M, z25.s, z15.s\n" + "fmax z24.s, p3/M, z24.s, z15.s\n" "fmin z28.s, p3/M, z28.s, z14.s\n" "st1w { z28.s }, p0, [x16, x22, LSL #2]\n" - "mov z28.d, z13.d\n" - "addvl x16, x16, #1\n" "fmin z27.s, p3/M, z27.s, z14.s\n" - "st1w { z27.s }, p0, [x26]\n" - "mov z27.d, z13.d\n" + "addvl x16, x16, #1\n" "fmin z26.s, p3/M, z26.s, z14.s\n" - "st1w { z26.s }, p0, [x26, x17, LSL #2]\n" - "mov z26.d, z13.d\n" + "st1w { z27.s }, p0, [x25]\n" "fmin z25.s, p3/M, z25.s, z14.s\n" - "st1w { z25.s }, p0, [x26, x23, LSL #2]\n" - "mov z25.d, z13.d\n" - "fmax z24.s, p3/M, z24.s, z15.s\n" + "fmin z24.s, p3/M, z24.s, z14.s\n" + "st1w { z26.s }, p0, [x25, x17, LSL #2]\n" "fmax z23.s, p3/M, z23.s, z15.s\n" + "st1w { z25.s }, p0, [x25, x26, LSL #2]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" "fmax z21.s, p3/M, z21.s, z15.s\n" - "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z24.s }, p0, [x26, x22, LSL #2]\n" - "mov z24.d, z13.d\n" - "addvl x26, x26, #1\n" + "st1w { z24.s }, p0, [x25, x22, LSL #2]\n" + "addvl x25, x25, #1\n" "fmin z23.s, p3/M, z23.s, z14.s\n" - "st1w { z23.s }, p0, [x25]\n" - "mov z23.d, z13.d\n" + "st1w { z23.s }, p0, [x24]\n" "fmin z22.s, p3/M, z22.s, z14.s\n" - "st1w { z22.s }, p0, [x25, x17, LSL #2]\n" - "mov z22.d, z13.d\n" "fmin z21.s, p3/M, z21.s, z14.s\n" - "st1w { z21.s }, p0, [x25, x23, LSL #2]\n" - "mov z21.d, z13.d\n" + "st1w { z22.s }, p0, [x24, x17, LSL #2]\n" "fmax z20.s, p3/M, z20.s, z15.s\n" "fmax z19.s, p3/M, z19.s, z15.s\n" + "st1w { z21.s }, p0, [x24, x26, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" "fmax z17.s, p3/M, z17.s, z15.s\n" + "fmax z16.s, p3/M, z16.s, z15.s\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "st1w { z20.s }, p0, [x25, x22, LSL #2]\n" - "mov z20.d, z13.d\n" - "addvl x25, x25, #1\n" + "st1w { z20.s }, p0, [x24, x22, LSL #2]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "st1w { z19.s }, p0, [x24]\n" - "mov z19.d, z13.d\n" + "addvl x24, x24, #1\n" "fmin z18.s, p3/M, z18.s, z14.s\n" - "st1w { z18.s }, p0, [x24, x17, LSL #2]\n" - "mov z18.d, z13.d\n" + "st1w { z19.s }, p0, [x23]\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "st1w { z17.s }, p0, [x24, x23, LSL #2]\n" - "mov z17.d, z13.d\n" - "fmax z16.s, p3/M, z16.s, z15.s\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z16.s }, p0, [x24, x22, LSL #2]\n" - "mov z16.d, z13.d\n" - "addvl x24, x24, #1\n" + "st1w { z18.s }, p0, [x23, x17, LSL #2]\n" + "st1w { z17.s }, p0, [x23, x26, LSL #2]\n" + "st1w { z16.s }, p0, [x23, x22, LSL #2]\n" + "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x2, #0x1\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x3, x3, #0x1\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" "cmp x3, x19\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" "csel x3, x3, XZR, LT\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" "csel x2, x2, x21, LT\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n" "cmp x2, x20\n" "fmla z31.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x11]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" "fmla z29.s, p3/M, z7.s, z12.s\n" "fmla z26.s, p3/M, z5.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" "fmla z22.s, p3/M, z2.s, z12.s\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" "fmla z26.s, p3/M, z7.s, z9.s\n" @@ -486,8 +454,8 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.s, p3/M, z4.s, z9.s\n" "fmla z21.s, p3/M, z3.s, z9.s\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" "fmla z30.s, p3/M, z0.s, z12.s\n" @@ -638,43 +606,43 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "st1w { z30.s }, p0, [x16, x17, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z14.s\n" "fmax z27.s, p3/M, z27.s, z15.s\n" - "st1w { z29.s }, p0, [x16, x23, LSL #2]\n" + "st1w { z29.s }, p0, [x16, x26, LSL #2]\n" "fmax z26.s, p3/M, z26.s, z15.s\n" "st1w { z28.s }, p0, [x16, x22, LSL #2]\n" "fmin z27.s, p3/M, z27.s, z14.s\n" "fmax z25.s, p3/M, z25.s, z15.s\n" - "st1w { z27.s }, p0, [x26]\n" + "st1w { z27.s }, p0, [x25]\n" "fmin z26.s, p3/M, z26.s, z14.s\n" "fmin z25.s, p3/M, z25.s, z14.s\n" - "st1w { z26.s }, p0, [x26, x17, LSL #2]\n" + "st1w { z26.s }, p0, [x25, x17, LSL #2]\n" "fmax z24.s, p3/M, z24.s, z15.s\n" "fmax z23.s, p3/M, z23.s, z15.s\n" - "st1w { z25.s }, p0, [x26, x23, LSL #2]\n" + "st1w { z25.s }, p0, [x25, x26, LSL #2]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" "fmax z21.s, p3/M, z21.s, z15.s\n" "fmax z20.s, p3/M, z20.s, z15.s\n" "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z24.s }, p0, [x26, x22, LSL #2]\n" + "st1w { z24.s }, p0, [x25, x22, LSL #2]\n" "fmin z23.s, p3/M, z23.s, z14.s\n" "fmin z22.s, p3/M, z22.s, z14.s\n" - "st1w { z23.s }, p0, [x25]\n" + "st1w { z23.s }, p0, [x24]\n" "fmin z21.s, p3/M, z21.s, z14.s\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "st1w { z22.s }, p0, [x25, x17, LSL #2]\n" + "st1w { z22.s }, p0, [x24, x17, LSL #2]\n" "fmax z19.s, p3/M, z19.s, z15.s\n" - "st1w { z21.s }, p0, [x25, x23, LSL #2]\n" + "st1w { z21.s }, p0, [x24, x26, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" "fmax z17.s, p3/M, z17.s, z15.s\n" - "st1w { z20.s }, p0, [x25, x22, LSL #2]\n" + "st1w { z20.s }, p0, [x24, x22, LSL #2]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "st1w { z19.s }, p0, [x24]\n" + "st1w { z19.s }, p0, [x23]\n" "fmin z18.s, p3/M, z18.s, z14.s\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "st1w { z18.s }, p0, [x24, x17, LSL #2]\n" + "st1w { z18.s }, p0, [x23, x17, LSL #2]\n" "fmax z16.s, p3/M, z16.s, z15.s\n" - "st1w { z17.s }, p0, [x24, x23, LSL #2]\n" + "st1w { z17.s }, p0, [x23, x26, LSL #2]\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z16.s }, p0, [x24, x22, LSL #2]\n" + "st1w { z16.s }, p0, [x23, x22, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -685,4 +653,4 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 22e12a7b9a..158d44046c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -98,644 +98,538 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x5, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n" - "add x7, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x8, ALL, MUL #2\n" + "mov x13, #0x0\n" "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "mov x17, #0x0\n" - "ld1w { z13.s }, p3/Z, [x6]\n" - "mov z31.d, z13.d\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" - "cntw x16\n" - "mov z30.d, z13.d\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" - "sub x15, XZR, x16\n" - "mov z29.d, z13.d\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" + "cntw x12\n" + "ld1w { z13.s }, p3/Z, [x15]\n" + "sub x11, XZR, x12\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z28.d, z13.d\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" - "cmp x16, %x[n_channels]\n" - "mov z27.d, z13.d\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" - "mov z26.d, z13.d\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" - "mov z25.d, z13.d\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" - "addvl x6, x6, #16\n" - "mov z24.d, z13.d\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "mov z23.d, z13.d\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" - "mov z22.d, z13.d\n" - "ldp x14, x13, [x7, #0x0]\n" - "mov z21.d, z13.d\n" - "ldp x12, x11, [x7, #0x10]\n" - "mov z20.d, z13.d\n" - "ld1w { z9.s }, p2/Z, [x14, x17, LSL #2]\n" - "mov z19.d, z13.d\n" - "mov z18.d, z13.d\n" - "prfm pldl1keep, [x14, x8]\n" - "mov z17.d, z13.d\n" - "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n" - "mov z16.d, z13.d\n" - "prfm pldl1keep, [x13, x8]\n" - "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n" - "prfm pldl1keep, [x12, x8]\n" - "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n" - "prfm pldl1keep, [x11, x8]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x7, #0x20]\n" - "whilelt p1.s, x16, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x21, [x7, #0x28]\n" - "incw x15\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x20, [x7, #0x30]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x20]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" + "incw x11\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x20, x8]\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" - "ldr x19, [x7, #0x38]\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" - "ldr x10, [x7, #0x40]\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x19, x8]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.s, p3/M, z5.s, z12.s\n" - "ldr x9, [x7, #0x48]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ldr x28, [x7, #0x50]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "ldr x27, [x7, #0x58]\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x27, x8]\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" - "ldr x26, [x7, #0x60]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.s, p3/M, z7.s, z9.s\n" - "ldr x25, [x7, #0x68]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.s, p3/M, z6.s, z9.s\n" - "ldr x24, [x7, #0x70]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x26, x8]\n" "fmla z22.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z21.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x24, x8]\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x7, #0x78]\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "ldr x14, [x7, #0x80]\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x14, x8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z26.s, p3/M, z8.s, z10.s\n" - "ldr x13, [x7, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.s, p3/M, z7.s, z10.s\n" - "ldr x12, [x7, #0x90]\n" + "ld1w { z13.s }, p3/Z, [x15]\n" "fmla z24.s, p3/M, z6.s, z10.s\n" - "ldr x11, [x7, #0x98]\n" "fmla z22.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" "fmla z21.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z20.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x11, x8]\n" "fmla z18.s, p3/M, z2.s, z10.s\n" - "ldr x22, [x7, #0xa0]\n" "fmla z17.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x7, #0xa8]\n" "fmla z16.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" "fmla z28.s, p3/M, z5.s, z12.s\n" - "ldr x20, [x7, #0xb0]\n" "fmla z24.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z6.s, z11.s\n" - "ldr x19, [x7, #0xb8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x20, x8]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x19, x8]\n" "fmla z29.s, p3/M, z3.s, z10.s\n" - "ldr x10, [x7, #0xc0]\n" "fmla z27.s, p3/M, z2.s, z10.s\n" - "ldr x9, [x7, #0xc8]\n" "fmla z26.s, p3/M, z1.s, z10.s\n" - "ldr x28, [x7, #0xd0]\n" "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z20.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ldr x27, [x7, #0xd8]\n" "fmla z26.s, p3/M, z2.s, z12.s\n" - "ldr x26, [x7, #0xe0]\n" "fmla z25.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x7, #0xe8]\n" "fmla z24.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z19.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x27, x8]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z10.s\n" - "prfm pldl1keep, [x26, x8]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z27.s, p3/M, z4.s, z10.s\n" - "ldr x24, [x7, #0xf0]\n" "fmla z26.s, p3/M, z3.s, z10.s\n" - "ldr x23, [x7, #0xf8]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x14, [x7, #0x100]\n" "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z17.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x24, x8]\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x14, x8]\n" "fmla z25.s, p3/M, z5.s, z12.s\n" - "ldr x13, [x7, #0x108]\n" "fmla z24.s, p3/M, z4.s, z12.s\n" - "ldr x12, [x7, #0x110]\n" "fmla z21.s, p3/M, z2.s, z12.s\n" - "ldr x11, [x7, #0x118]\n" "fmla z20.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z29.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x8]\n" - "addvl x8, x8, #1\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.s, p3/M, z6.s, z11.s\n" - "ldr x22, [x5, #0x0]\n" "fmla z23.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x5, #0x8]\n" "fmla z22.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x5, #0x10]\n" "fmla z19.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x5, #0x18]\n" "fmla z18.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z13.s }, p3/Z, [x6]\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z10.s\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.s, p3/M, z3.s, z10.s\n" "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z25.s, p3/M, z8.s, z11.s\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z5.s, z11.s\n" "fmla z20.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z2.s, z11.s\n" "fmla z16.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z28.s, p3/M, z8.s, z12.s\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.s, p3/M, z5.s, z12.s\n" "fmla z20.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z27.s, p3/M, z6.s, z10.s\n" "fmla z23.s, p3/M, z3.s, z10.s\n" "fmla z19.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z22.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z6.s, z11.s\n" "fmla z23.s, p3/M, z8.s, z11.s\n" "fmla z19.s, p3/M, z5.s, z11.s\n" "fmla z18.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z12.s\n" "fmla z20.s, p3/M, z5.s, z12.s\n" "fmla z16.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z19.s, p3/M, z8.s, z10.s\n" "fmla z18.s, p3/M, z7.s, z10.s\n" "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z22.s, p3/M, z8.s, z11.s\n" "fmla z21.s, p3/M, z7.s, z11.s\n" "fmla z20.s, p3/M, z6.s, z11.s\n" "fmla z18.s, p3/M, z5.s, z11.s\n" "fmla z17.s, p3/M, z4.s, z11.s\n" "fmla z16.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z10.s\n" - "ldp x14, x13, [x7, #0x0]\n" + "ldp x10, x9, [x14, #0x0]\n" "fmla z18.s, p3/M, z8.s, z12.s\n" - "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n" "fmla z17.s, p3/M, z7.s, z12.s\n" "fmla z16.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x14, x8]\n" "fmla z27.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" "fmla z26.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n" - "incw x17\n" + "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n" + "incw x13\n" "fmla z29.s, p3/M, z5.s, z11.s\n" - "ldp x12, x11, [x7, #0x10]\n" - "whilelt p2.s, x17, %x[n_channels]\n" + "ldp x28, x27, [x14, #0x10]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z24.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x12, x16, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n" "fmla z23.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x12, x8]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z22.s, p3/M, z6.s, z12.s\n" - "prfm pldl1keep, [x11, x8]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" "fmla z19.s, p3/M, z4.s, z12.s\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" "fmla z18.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x11, x16, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n" "fmla z21.s, p3/M, z8.s, z10.s\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z20.s, p3/M, z7.s, z10.s\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" "fmla z17.s, p3/M, z5.s, z10.s\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z16.s, p3/M, z4.s, z10.s\n" - "ld1w { z10.s }, p1/Z, [x13, x16, LSL #2]\n" - "incw x16\n" + "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n" + "incw x12\n" "fmax z31.s, p3/M, z31.s, z15.s\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" - "addvl x6, x6, #16\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z30.s, p3/M, z30.s, z15.s\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "cmp x16, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.s, p3/M, z29.s, z15.s\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.s, p3/M, z28.s, z15.s\n" "fmax z27.s, p3/M, z27.s, z15.s\n" "fmin z31.s, p3/M, z31.s, z14.s\n" - "st1w { z31.s }, p0, [x22, x15, LSL #2]\n" - "mov z31.d, z13.d\n" + "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z14.s\n" - "ldr x22, [x5, #0x20]\n" "fmin z29.s, p3/M, z29.s, z14.s\n" - "st1w { z30.s }, p0, [x21, x15, LSL #2]\n" - "mov z30.d, z13.d\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.s, p3/M, z28.s, z14.s\n" - "st1w { z29.s }, p0, [x20, x15, LSL #2]\n" - "mov z29.d, z13.d\n" - "ldr x21, [x5, #0x28]\n" + "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" "fmin z27.s, p3/M, z27.s, z14.s\n" - "ldr x20, [x5, #0x30]\n" "fmax z26.s, p3/M, z26.s, z15.s\n" - "st1w { z28.s }, p0, [x19, x15, LSL #2]\n" - "mov z28.d, z13.d\n" - "ldr x19, [x5, #0x38]\n" + "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" "fmax z25.s, p3/M, z25.s, z15.s\n" - "st1w { z27.s }, p0, [x22, x15, LSL #2]\n" - "mov z27.d, z13.d\n" - "ldr x22, [x5, #0x40]\n" - "fmin z26.s, p3/M, z26.s, z14.s\n" - "st1w { z26.s }, p0, [x21, x15, LSL #2]\n" - "mov z26.d, z13.d\n" - "fmin z25.s, p3/M, z25.s, z14.s\n" - "ldr x21, [x5, #0x48]\n" + "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" "fmax z24.s, p3/M, z24.s, z15.s\n" - "st1w { z25.s }, p0, [x20, x15, LSL #2]\n" - "mov z25.d, z13.d\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.s, p3/M, z23.s, z15.s\n" - "ldr x20, [x5, #0x50]\n" + "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" + "fmin z26.s, p3/M, z26.s, z14.s\n" + "ldr x20, [x16, #0x30]\n" + "fmin z25.s, p3/M, z25.s, z14.s\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z24.s }, p0, [x19, x15, LSL #2]\n" - "mov z24.d, z13.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.s, p3/M, z23.s, z14.s\n" - "ldr x19, [x5, #0x58]\n" + "st1w { z26.s }, p0, [x21, x11, LSL #2]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" - "st1w { z23.s }, p0, [x22, x15, LSL #2]\n" - "mov z23.d, z13.d\n" + "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" "fmax z21.s, p3/M, z21.s, z15.s\n" - "ldr x22, [x5, #0x60]\n" - "fmin z22.s, p3/M, z22.s, z14.s\n" - "st1w { z22.s }, p0, [x21, x15, LSL #2]\n" - "mov z22.d, z13.d\n" - "fmin z21.s, p3/M, z21.s, z14.s\n" - "ldr x21, [x5, #0x68]\n" + "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" "fmax z20.s, p3/M, z20.s, z15.s\n" - "st1w { z21.s }, p0, [x20, x15, LSL #2]\n" - "mov z21.d, z13.d\n" + "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" "fmax z19.s, p3/M, z19.s, z15.s\n" - "ldr x20, [x5, #0x70]\n" + "ldr x21, [x16, #0x48]\n" + "fmin z22.s, p3/M, z22.s, z14.s\n" + "ldr x20, [x16, #0x50]\n" + "fmin z21.s, p3/M, z21.s, z14.s\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "st1w { z20.s }, p0, [x19, x15, LSL #2]\n" - "mov z20.d, z13.d\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "ldr x19, [x5, #0x78]\n" + "st1w { z22.s }, p0, [x21, x11, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" - "st1w { z19.s }, p0, [x22, x15, LSL #2]\n" - "mov z19.d, z13.d\n" + "st1w { z21.s }, p0, [x20, x11, LSL #2]\n" "fmax z17.s, p3/M, z17.s, z15.s\n" + "st1w { z20.s }, p0, [x19, x11, LSL #2]\n" + "fmax z16.s, p3/M, z16.s, z15.s\n" + "st1w { z19.s }, p0, [x22, x11, LSL #2]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.s, p3/M, z18.s, z14.s\n" - "st1w { z18.s }, p0, [x21, x15, LSL #2]\n" - "mov z18.d, z13.d\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "st1w { z17.s }, p0, [x20, x15, LSL #2]\n" - "mov z17.d, z13.d\n" - "fmax z16.s, p3/M, z16.s, z15.s\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z16.s }, p0, [x19, x15, LSL #2]\n" - "mov z16.d, z13.d\n" + "st1w { z18.s }, p0, [x21, x11, LSL #2]\n" + "st1w { z17.s }, p0, [x20, x11, LSL #2]\n" + "st1w { z16.s }, p0, [x19, x11, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x7, #0x20]\n" - "incw x15\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x21, [x7, #0x28]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x20]\n" + "incw x11\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x20, [x7, #0x30]\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x20, x8]\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" - "ldr x19, [x7, #0x38]\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" - "ldr x10, [x7, #0x40]\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x19, x8]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.s, p3/M, z5.s, z12.s\n" - "ldr x9, [x7, #0x48]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ldr x28, [x7, #0x50]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "ldr x27, [x7, #0x58]\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x27, x8]\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" - "ldr x26, [x7, #0x60]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.s, p3/M, z7.s, z9.s\n" - "ldr x25, [x7, #0x68]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.s, p3/M, z6.s, z9.s\n" - "ldr x24, [x7, #0x70]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x26, x8]\n" "fmla z22.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z21.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x24, x8]\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x7, #0x78]\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "ldr x14, [x7, #0x80]\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x14, x8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z26.s, p3/M, z8.s, z10.s\n" - "ldr x13, [x7, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.s, p3/M, z7.s, z10.s\n" - "ldr x12, [x7, #0x90]\n" "fmla z24.s, p3/M, z6.s, z10.s\n" - "ldr x11, [x7, #0x98]\n" "fmla z22.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" "fmla z21.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z20.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x11, x8]\n" "fmla z18.s, p3/M, z2.s, z10.s\n" - "ldr x22, [x7, #0xa0]\n" "fmla z17.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x7, #0xa8]\n" "fmla z16.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" "fmla z28.s, p3/M, z5.s, z12.s\n" - "ldr x20, [x7, #0xb0]\n" "fmla z24.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z6.s, z11.s\n" - "ldr x19, [x7, #0xb8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x20, x8]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x19, x8]\n" "fmla z29.s, p3/M, z3.s, z10.s\n" - "ldr x10, [x7, #0xc0]\n" "fmla z27.s, p3/M, z2.s, z10.s\n" - "ldr x9, [x7, #0xc8]\n" "fmla z26.s, p3/M, z1.s, z10.s\n" - "ldr x28, [x7, #0xd0]\n" "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z20.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ldr x27, [x7, #0xd8]\n" "fmla z26.s, p3/M, z2.s, z12.s\n" - "ldr x26, [x7, #0xe0]\n" "fmla z25.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x7, #0xe8]\n" "fmla z24.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z19.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x27, x8]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z10.s\n" - "prfm pldl1keep, [x26, x8]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z27.s, p3/M, z4.s, z10.s\n" - "ldr x24, [x7, #0xf0]\n" "fmla z26.s, p3/M, z3.s, z10.s\n" - "ldr x23, [x7, #0xf8]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x14, [x7, #0x100]\n" "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z17.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x24, x8]\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x14, x8]\n" "fmla z25.s, p3/M, z5.s, z12.s\n" - "ldr x13, [x7, #0x108]\n" "fmla z24.s, p3/M, z4.s, z12.s\n" - "ldr x12, [x7, #0x110]\n" "fmla z21.s, p3/M, z2.s, z12.s\n" - "ldr x11, [x7, #0x118]\n" "fmla z20.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z29.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x8]\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.s, p3/M, z6.s, z11.s\n" - "ldr x22, [x5, #0x0]\n" "fmla z23.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x5, #0x8]\n" "fmla z22.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x5, #0x10]\n" "fmla z19.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x5, #0x18]\n" "fmla z18.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z10.s\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.s, p3/M, z3.s, z10.s\n" "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z25.s, p3/M, z8.s, z11.s\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z5.s, z11.s\n" "fmla z20.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z2.s, z11.s\n" "fmla z16.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z28.s, p3/M, z8.s, z12.s\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.s, p3/M, z5.s, z12.s\n" "fmla z20.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z27.s, p3/M, z6.s, z10.s\n" "fmla z23.s, p3/M, z3.s, z10.s\n" "fmla z19.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z22.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z6.s, z11.s\n" "fmla z23.s, p3/M, z8.s, z11.s\n" "fmla z19.s, p3/M, z5.s, z11.s\n" "fmla z18.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z12.s\n" "fmla z20.s, p3/M, z5.s, z12.s\n" "fmla z16.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z19.s, p3/M, z8.s, z10.s\n" "fmla z18.s, p3/M, z7.s, z10.s\n" "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z22.s, p3/M, z8.s, z11.s\n" "fmla z21.s, p3/M, z7.s, z11.s\n" "fmla z20.s, p3/M, z6.s, z11.s\n" "fmla z18.s, p3/M, z5.s, z11.s\n" "fmla z17.s, p3/M, z4.s, z11.s\n" "fmla z16.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z10.s\n" "fmla z18.s, p3/M, z8.s, z12.s\n" "fmla z17.s, p3/M, z7.s, z12.s\n" "fmla z16.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z10.s\n" "fmla z27.s, p3/M, z1.s, z10.s\n" "fmla z26.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z11.s\n" "fmla z28.s, p3/M, z4.s, z11.s\n" "fmla z25.s, p3/M, z2.s, z11.s\n" @@ -753,68 +647,68 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "fmax z29.s, p3/M, z29.s, z15.s\n" "fmax z28.s, p3/M, z28.s, z15.s\n" "fmin z31.s, p3/M, z31.s, z14.s\n" - "st1w { z31.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z14.s\n" "fmin z29.s, p3/M, z29.s, z14.s\n" - "ldr x22, [x5, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.s, p3/M, z28.s, z14.s\n" - "st1w { z30.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" "fmax z27.s, p3/M, z27.s, z15.s\n" "fmax z26.s, p3/M, z26.s, z15.s\n" - "st1w { z29.s }, p0, [x20, x15, LSL #2]\n" + "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" "fmax z25.s, p3/M, z25.s, z15.s\n" - "st1w { z28.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" "fmax z24.s, p3/M, z24.s, z15.s\n" - "ldr x21, [x5, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.s, p3/M, z23.s, z15.s\n" - "ldr x20, [x5, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmin z27.s, p3/M, z27.s, z14.s\n" - "ldr x19, [x5, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z26.s, p3/M, z26.s, z14.s\n" - "st1w { z27.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" "fmin z25.s, p3/M, z25.s, z14.s\n" "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z26.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z26.s }, p0, [x21, x11, LSL #2]\n" "fmin z23.s, p3/M, z23.s, z14.s\n" - "ldr x22, [x5, #0x40]\n" + "ldr x22, [x16, #0x40]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" - "ldr x21, [x5, #0x48]\n" + "ldr x21, [x16, #0x48]\n" "fmax z21.s, p3/M, z21.s, z15.s\n" - "st1w { z25.s }, p0, [x20, x15, LSL #2]\n" + "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" "fmax z20.s, p3/M, z20.s, z15.s\n" - "st1w { z24.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" "fmax z19.s, p3/M, z19.s, z15.s\n" - "st1w { z23.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" "fmin z22.s, p3/M, z22.s, z14.s\n" - "ldr x20, [x5, #0x50]\n" + "ldr x20, [x16, #0x50]\n" "fmin z21.s, p3/M, z21.s, z14.s\n" - "ldr x19, [x5, #0x58]\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "ldr x22, [x5, #0x60]\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "st1w { z22.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z22.s }, p0, [x21, x11, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" - "st1w { z21.s }, p0, [x20, x15, LSL #2]\n" + "st1w { z21.s }, p0, [x20, x11, LSL #2]\n" "fmax z17.s, p3/M, z17.s, z15.s\n" - "st1w { z20.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z20.s }, p0, [x19, x11, LSL #2]\n" "fmax z16.s, p3/M, z16.s, z15.s\n" - "st1w { z19.s }, p0, [x22, x15, LSL #2]\n" - "ldr x21, [x5, #0x68]\n" + "st1w { z19.s }, p0, [x22, x11, LSL #2]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.s, p3/M, z18.s, z14.s\n" - "ldr x20, [x5, #0x70]\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "ldr x19, [x5, #0x78]\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z18.s }, p0, [x21, x15, LSL #2]\n" - "st1w { z17.s }, p0, [x20, x15, LSL #2]\n" - "st1w { z16.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z18.s }, p0, [x21, x11, LSL #2]\n" + "st1w { z17.s }, p0, [x20, x11, LSL #2]\n" + "st1w { z16.s }, p0, [x19, x11, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 2119c06965..28d44d0b5f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst +class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 78e67e1be1..ac33dcbce5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,318 +88,250 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( __asm__ __volatile__( "ptrue p3.b\n" - "mov x5, #0x0\n" - "mov x6, #0x0\n" + "mov x7, #0x0\n" + "mov x8, #0x0\n" "1:" // Tile loop - "str x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "mov x21, #0x4\n" - "str x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "cntb x7\n" - "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" - "add x7, x7, XZR, LSL #4\n" - "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" - "cntb x17\n" - "ldr x16, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" - "cntb x15\n" - "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n" - "mul x19, x5, x20\n" // offset = tile_i * ld_input_row - "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" - "madd x19, x6, x16, x19\n" // offset += tile_j * ld_input_col - "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" - "mul x19, x19, x21\n" // offset *= kernel_stride * output_size - "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n" - "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) + "str x7, [%x[params_struct], %[offsetof_args_tile_i]]\n" + "mov x23, #0x4\n" + "str x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "mov x17, #0x2\n" + "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x15, #0x0\n" + "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" + "cntw x14\n" + "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" + "sub x12, XZR, x14\n" + "ldr x21, [%x[params_struct], %[offsetof_args_inptr]]\n" + "mul x19, x7, x22\n" // offset = tile_i * ld_input_row + "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" + "madd x19, x8, x13, x19\n" // offset += tile_j * ld_input_col + "ldr x11, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" + "mul x19, x19, x23\n" // offset *= kernel_stride * output_size + "ldr x10, [%x[params_struct], %[offsetof_args_outptr]]\n" + "add x21, x21, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "add x11, x14, x20, LSL #2\n" + "add x9, x21, x22, LSL #2\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "add x10, x11, x20, LSL #2\n" - "ld1w { z17.s }, p3/Z, [x8]\n" - "mov z31.d, z17.d\n" - "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" - "add x9, x10, x20, LSL #2\n" - "mov z30.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" - "add x28, x9, x20, LSL #2\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" - "add x27, x16, x16\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" - "add x26, x27, x16\n" - "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" - "add x25, x26, x16\n" - "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" - "add x17, x17, x16, LSL #4\n" - "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" - "add x15, x15, x27, LSL #4\n" - "cntb x24\n" - "prfm pldl1keep, [x10, x15]\n" - "prfm pldl1keep, [x14, x7]\n" - "add x24, x24, x26, LSL #4\n" - "prfm pldl1keep, [x14, x17]\n" - "cntb x23\n" - "prfm pldl1keep, [x14, x24]\n" - "add x23, x23, x25, LSL #4\n" - "mov x20, #0x2\n" - "prfm pldl1keep, [x14, x23]\n" - "prfm pldl1keep, [x11, x7]\n" - "mul x19, x5, x22\n" // offset = tile_i * ld_output_row - "prfm pldl1keep, [x11, x17]\n" - "madd x19, x6, x13, x19\n" // offset += tile_j * ld_output_col - "prfm pldl1keep, [x14, x15]\n" - "mul x19, x19, x20\n" // offset *= output_tile_size - "mov x21, #0x0\n" - "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "add x22, x12, x22, LSL #2\n" - "cntw x20\n" - "sub x19, XZR, x20\n" + "add x28, x9, x22, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x16]\n" + "add x27, x28, x22, LSL #2\n" + "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "add x26, x27, x22, LSL #2\n" + "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "add x25, x13, x13\n" + "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "add x24, x25, x13\n" + "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "add x23, x24, x13\n" + "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" + "mul x19, x7, x20\n" // offset = tile_i * ld_output_row + "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" + "madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col + "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" + "mul x19, x19, x17\n" // offset *= output_tile_size "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x14]\n" - "addvl x8, x8, #16\n" - "ld1w { z11.s }, p2/Z, [x14, x16, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" - "addvl x8, x8, #-6\n" - "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x14, x25, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x11]\n" - "ld1w { z15.s }, p2/Z, [x11, x16, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x14, x27, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x28, x25, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x21]\n" + "add x10, x10, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" + "add x22, x10, x20, LSL #2\n" + "ld1w { z12.s }, p2/Z, [x21, x24, LSL #2]\n" + "addvl x16, x16, #16\n" + "ld1w { z13.s }, p2/Z, [x21, x23, LSL #2]\n" + "cmp x14, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" + "addvl x16, x16, #-6\n" + "ld1w { z14.s }, p2/Z, [x9]\n" + "ld1w { z15.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x25, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x11, x24]\n" - "whilelt p1.s, x20, %x[n_channels]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x11, x23]\n" - "incw x19\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x11, x15]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "whilelt p1.s, x14, %x[n_channels]\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "incw x12\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x9, x7]\n" - "incw x21\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z17.s }, p3/Z, [x16]\n" + "incw x15\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x10, x7]\n" - "addvl x14, x14, #1\n" + "addvl x21, x21, #1\n" + "ld1w { z10.s }, p1/Z, [x21]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n" - "incw x20\n" + "incw x14\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" - "prfm pldl1keep, [x9, x17]\n" + "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x9]\n" - "addvl x11, x11, #1\n" + "ld1w { z14.s }, p2/Z, [x27]\n" + "addvl x9, x9, #1\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "prfm pldl1keep, [x10, x17]\n" - "prfm pldl1keep, [x9, x24]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "prfm pldl1keep, [x10, x24]\n" - "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x10]\n" + "ld1w { z15.s }, p2/Z, [x28]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "prfm pldl1keep, [x9, x23]\n" - "prfm pldl1keep, [x28, x7]\n" + "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" - "prfm pldl1keep, [x10, x23]\n" + "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n" + "addvl x28, x28, #1\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "addvl x10, x10, #1\n" + "ld1w { z9.s }, p1/Z, [x28, x25, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" - "prfm pldl1keep, [x28, x17]\n" + "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "prfm pldl1keep, [x9, x15]\n" - "prfm pldl1keep, [x28, x24]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "ld1w { z15.s }, p2/Z, [x28]\n" + "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n" - "prfm pldl1keep, [x28, x15]\n" + "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" + "fmla z29.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n" - "addvl x9, x9, #1\n" + "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n" + "addvl x27, x27, #1\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x28, x23]\n" - "fmla z29.s, p3/M, z6.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x21, x24, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n" - "prfm pldl1keep, [x10, x15]\n" + "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "fmla z29.s, p3/M, z7.s, z13.s\n" + "ld1w { z13.s }, p1/Z, [x21, x23, LSL #2]\n" "fmax z31.s, p3/M, z31.s, z19.s\n" - "ld1w { z10.s }, p1/Z, [x14]\n" "fmla z28.s, p3/M, z5.s, z14.s\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n" "fmax z30.s, p3/M, z30.s, z19.s\n" - "prfm pldl1keep, [x14, x7]\n" - "prfm pldl1keep, [x14, x17]\n" + "fmla z29.s, p3/M, z5.s, z16.s\n" + "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" "fmin z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x14, x26, LSL #2]\n" + "st1w { z31.s }, p0, [x10]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "fmla z29.s, p3/M, z5.s, z16.s\n" - "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n" - "whilelt p2.s, x21, %x[n_channels]\n" + "fmla z29.s, p3/M, z8.s, z15.s\n" + "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n" + "whilelt p2.s, x15, %x[n_channels]\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "prfm pldl1keep, [x14, x24]\n" - "addvl x28, x28, #1\n" + "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "addvl x26, x26, #1\n" "fmla z28.s, p3/M, z3.s, z16.s\n" - "ld1w { z13.s }, p1/Z, [x14, x25, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "fmla z29.s, p3/M, z8.s, z15.s\n" - "prfm pldl1keep, [x14, x23]\n" - "prfm pldl1keep, [x11, x7]\n" - "fmla z28.s, p3/M, z7.s, z14.s\n" - "ld1w { z14.s }, p1/Z, [x11]\n" - "prfm pldl1keep, [x11, x17]\n" + "ld1w { z16.s }, p1/Z, [x21, x25, LSL #2]\n" + "cmp x14, %x[n_channels]\n" "fmax z29.s, p3/M, z29.s, z19.s\n" - "ld1w { z16.s }, p1/Z, [x14, x27, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ld1w { z15.s }, p1/Z, [x11, x16, LSL #2]\n" - "prfm pldl1keep, [x14, x15]\n" + "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "st1w { z30.s }, p0, [x10, x11, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z14.s\n" + "ld1w { z14.s }, p1/Z, [x9]\n" + "addvl x10, x10, #1\n" "fmin z29.s, p3/M, z29.s, z18.s\n" - "st1w { z31.s }, p0, [x12]\n" + "st1w { z29.s }, p0, [x22]\n" + "fmla z28.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p1/Z, [x9, x13, LSL #2]\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x14, x16, LSL #2]\n" - "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x21, x13, LSL #2]\n" + "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" "fmax z28.s, p3/M, z28.s, z19.s\n" - "st1w { z29.s }, p0, [x22]\n" - "addvl x12, x12, #1\n" + "addvl x16, x16, #16\n" + "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z17.s }, p3/Z, [x8]\n" - "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" - "mov z31.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" - "mov z30.d, z17.d\n" - "st1w { z28.s }, p0, [x22, x13, LSL #2]\n" + "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" + "addvl x16, x16, #-6\n" + "st1w { z28.s }, p0, [x22, x11, LSL #2]\n" "addvl x22, x22, #1\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" - "addvl x8, x8, #16\n" - "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" - "addvl x8, x8, #-6\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x11, x24]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x11, x23]\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x11, x15]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x9, x7]\n" - "prfm pldl1keep, [x10, x7]\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "add x21, x7, #0x1\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" + "add x8, x8, #0x1\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x9, x17]\n" + "cmp x8, x19\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" - "prfm pldl1keep, [x10, x17]\n" + "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n" + "csel x8, x8, XZR, LT\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n" + "csel x7, x7, x21, LT\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x9]\n" - "prfm pldl1keep, [x9, x24]\n" + "ld1w { z14.s }, p2/Z, [x27]\n" + "cmp x7, x20\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "prfm pldl1keep, [x10, x24]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n" - "prfm pldl1keep, [x9, x23]\n" + "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x10]\n" + "ld1w { z15.s }, p2/Z, [x28]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "prfm pldl1keep, [x28, x7]\n" - "prfm pldl1keep, [x10, x23]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" - "prfm pldl1keep, [x28, x17]\n" + "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x9, x15]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "prfm pldl1keep, [x28, x24]\n" - "prfm pldl1keep, [x28, x15]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "ld1w { z15.s }, p2/Z, [x28]\n" + "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n" - "prfm pldl1keep, [x28, x23]\n" + "fmla z29.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "add x21, x5, #0x1\n" - "fmla z28.s, p3/M, z5.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "add x6, x6, #0x1\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z13.s\n" - "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmax z31.s, p3/M, z31.s, z19.s\n" - "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "cmp x6, x19\n" + "fmla z28.s, p3/M, z5.s, z14.s\n" + "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n" + "fmax z30.s, p3/M, z30.s, z19.s\n" "fmla z29.s, p3/M, z5.s, z16.s\n" - "fmla z28.s, p3/M, z3.s, z16.s\n" - "csel x6, x6, XZR, LT\n" "fmin z31.s, p3/M, z31.s, z18.s\n" - "st1w { z31.s }, p0, [x12]\n" - "fmla z28.s, p3/M, z7.s, z14.s\n" - "csel x5, x5, x21, LT\n" + "st1w { z31.s }, p0, [x10]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" "fmla z29.s, p3/M, z8.s, z15.s\n" - "cmp x5, x20\n" - "fmax z30.s, p3/M, z30.s, z19.s\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" + "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" + "st1w { z30.s }, p0, [x10, x11, LSL #2]\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" "fmax z29.s, p3/M, z29.s, z19.s\n" + "fmla z28.s, p3/M, z7.s, z14.s\n" "fmin z29.s, p3/M, z29.s, z18.s\n" "st1w { z29.s }, p0, [x22]\n" + "fmla z28.s, p3/M, z6.s, z15.s\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" "fmax z28.s, p3/M, z28.s, z19.s\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "st1w { z28.s }, p0, [x22, x13, LSL #2]\n" + "st1w { z28.s }, p0, [x22, x11, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 8555cfea7c..829b0ff2c7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -89,309 +89,249 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" - "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x12, ALL, MUL #2\n" + "mov x13, #0x0\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "mov x11, #0x0\n" - "ldp x10, x9, [x19, #0x0]\n" - "cntw x28\n" - "ldp x27, x26, [x19, #0x10]\n" - "sub x25, XZR, x28\n" - "ld1w { z17.s }, p3/Z, [x14]\n" - "mov z31.d, z17.d\n" - "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" + "cntw x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z30.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" - "cmp x28, %x[n_channels]\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" - "addvl x14, x14, #16\n" - "ldp x21, x20, [x13, #0x0]\n" - "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" - "addvl x14, x14, #-6\n" - "ld1w { z9.s }, p2/Z, [x21, x11, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z10.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" - "ldp x24, x23, [x13, #0x10]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldp x20, x19, [x13, #0x30]\n" - "ld1w { z11.s }, p2/Z, [x24, x11, LSL #2]\n" - "prfm pldl1keep, [x24, x12]\n" - "ld1w { z12.s }, p2/Z, [x23, x11, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" - "ld1w { z14.s }, p2/Z, [x21, x11, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z15.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" - "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z17.s }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x19, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x13, #0x40]\n" - "whilelt p1.s, x28, %x[n_channels]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "ldr x21, [x13, #0x48]\n" - "incw x25\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x20, [x13, #0x50]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x40]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "ldr x25, [x14, #0x48]\n" + "incw x9\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x24, [x14, #0x50]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x22, x12]\n" - "ldr x19, [x13, #0x58]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "ldr x21, [x13, #0x60]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "ldr x20, [x13, #0x68]\n" - "ldr x19, [x13, #0x70]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ldr x19, [x13, #0x78]\n" - "ldr x21, [x13, #0x80]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "ldr x20, [x13, #0x88]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x24, [x13, #0x90]\n" + "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x24, x12]\n" - "ldr x23, [x13, #0x98]\n" + "ld1w { z17.s }, p3/Z, [x15]\n" "fmla z29.s, p3/M, z6.s, z15.s\n" - "ldr x22, [x13, #0xa0]\n" - "fmax z31.s, p3/M, z31.s, z19.s\n" - "ldr x21, [x13, #0xa8]\n" - "fmla z28.s, p3/M, z5.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "fmin z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" + "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "fmla z28.s, p3/M, z5.s, z14.s\n" + "fmax z31.s, p3/M, z31.s, z19.s\n" + "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z13.s\n" - "ldr x20, [x13, #0xb0]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" "fmax z30.s, p3/M, z30.s, z19.s\n" - "ldr x19, [x13, #0xb8]\n" - "ldr x22, [x13, #0xc0]\n" - "fmla z28.s, p3/M, z3.s, z16.s\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "incw x13\n" "fmla z29.s, p3/M, z5.s, z16.s\n" - "prfm pldl1keep, [x20, x12]\n" + "ldp x26, x25, [x14, #0x0]\n" + "whilelt p2.s, x13, %x[n_channels]\n" + "fmin z31.s, p3/M, z31.s, z18.s\n" + "ldp x24, x23, [x14, #0x10]\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" - "fmla z28.s, p3/M, z7.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" + "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z15.s\n" - "prfm pldl1keep, [x22, x12]\n" - "incw x11\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ldp x21, x20, [x13, #0x0]\n" - "whilelt p2.s, x11, %x[n_channels]\n" + "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z14.s\n" + "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n" "fmax z29.s, p3/M, z29.s, z19.s\n" - "ldp x24, x23, [x13, #0x10]\n" - "addvl x12, x12, #1\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z9.s }, p1/Z, [x21, x28, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z14.s }, p1/Z, [x21, x12, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p1/Z, [x20, x12, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x19, x12, LSL #2]\n" "fmin z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z10.s }, p1/Z, [x20, x28, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n" + "incw x12\n" "fmax z28.s, p3/M, z28.s, z19.s\n" - "ld1w { z11.s }, p1/Z, [x24, x28, LSL #2]\n" - "prfm pldl1keep, [x24, x12]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" + "cmp x12, %x[n_channels]\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x23, x28, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldp x20, x19, [x13, #0x30]\n" - "st1w { z31.s }, p0, [x10, x25, LSL #2]\n" - "ld1w { z13.s }, p1/Z, [x22, x28, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" - "ld1w { z14.s }, p1/Z, [x21, x28, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z15.s }, p1/Z, [x20, x28, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" - "ld1w { z16.s }, p1/Z, [x19, x28, LSL #2]\n" - "incw x28\n" - "prfm pldl1keep, [x19, x12]\n" - "cmp x28, %x[n_channels]\n" - "st1w { z30.s }, p0, [x9, x25, LSL #2]\n" - "st1w { z29.s }, p0, [x27, x25, LSL #2]\n" - "st1w { z28.s }, p0, [x26, x25, LSL #2]\n" - "ld1w { z17.s }, p3/Z, [x14]\n" - "mov z31.d, z17.d\n" - "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" - "mov z30.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" - "addvl x14, x14, #16\n" - "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" - "addvl x14, x14, #-6\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x13, #0x40]\n" - "incw x25\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "ldr x21, [x13, #0x48]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x40]\n" + "incw x9\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "ldr x25, [x14, #0x48]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x20, [x13, #0x50]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x22, x12]\n" - "ldr x19, [x13, #0x58]\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x24, [x14, #0x50]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "ldr x21, [x13, #0x60]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "ldr x20, [x13, #0x68]\n" - "ldr x19, [x13, #0x70]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ldr x19, [x13, #0x78]\n" - "ldr x21, [x13, #0x80]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "ldr x20, [x13, #0x88]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x24, [x13, #0x90]\n" + "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x24, x12]\n" - "ldr x23, [x13, #0x98]\n" "fmla z29.s, p3/M, z6.s, z15.s\n" - "ldr x22, [x13, #0xa0]\n" - "fmax z31.s, p3/M, z31.s, z19.s\n" - "ldr x21, [x13, #0xa8]\n" - "fmla z28.s, p3/M, z5.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "fmin z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" + "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "fmla z28.s, p3/M, z5.s, z14.s\n" + "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n" + "fmax z31.s, p3/M, z31.s, z19.s\n" "fmla z29.s, p3/M, z7.s, z13.s\n" - "ldr x20, [x13, #0xb0]\n" "fmax z30.s, p3/M, z30.s, z19.s\n" - "ldr x19, [x13, #0xb8]\n" - "ldr x22, [x13, #0xc0]\n" - "fmla z28.s, p3/M, z3.s, z16.s\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "fmin z31.s, p3/M, z31.s, z18.s\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z16.s\n" - "prfm pldl1keep, [x20, x12]\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z15.s\n" - "prfm pldl1keep, [x22, x12]\n" - "st1w { z31.s }, p0, [x10, x25, LSL #2]\n" "fmla z28.s, p3/M, z6.s, z15.s\n" - "st1w { z30.s }, p0, [x9, x25, LSL #2]\n" "fmax z29.s, p3/M, z29.s, z19.s\n" "fmla z28.s, p3/M, z8.s, z11.s\n" "fmin z29.s, p3/M, z29.s, z18.s\n" - "st1w { z29.s }, p0, [x27, x25, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z19.s\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "st1w { z28.s }, p0, [x26, x25, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 6f1f187818..751874ffbf 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst +class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast<const float *const *>(input_ptrs), + reinterpret_cast<float *const *>(outptrs), + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast<const float *>(inptr), ld_input_row, ld_input_col, + static_cast<float *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast<const float *>(activation_min), + *static_cast<const float *>(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index edafe82770..ea8bbbd7e8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -114,22 +114,18 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x10, x20, x22, LSL #2\n" "ld1w { z16.s }, p3/Z, [x8]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" "add x9, x10, x22, LSL #2\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" "add x28, x9, x22, LSL #2\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" "add x27, x28, x22, LSL #2\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" "add x26, x15, x15\n" - "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" "add x25, x26, x15\n" - "mul x19, x5, x21\n" // offset = tile_i * ld_output_row + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" "add x24, x25, x15\n" + "mul x19, x5, x21\n" // offset = tile_i * ld_output_row "add x23, x24, x15\n" "madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col "mul x19, x19, x7\n" // offset *= output_tile_size @@ -150,14 +146,14 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1w { z14.s }, p2/Z, [x10]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z0.s, z5.s\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n" "whilelt p1.s, x16, %x[n_channels]\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" "incw x14\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" "ld1w { z0.s }, p3/Z, [x8]\n" "incw x17\n" "fmla z31.s, p3/M, z1.s, z6.s\n" @@ -334,29 +330,25 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "fmax z28.s, p3/M, z28.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" "st1w { z31.s }, p0, [x11]\n" - "mov z31.d, z16.d\n" "fmin z30.s, p3/M, z30.s, z17.s\n" + "fmin z29.s, p3/M, z29.s, z17.s\n" "st1w { z30.s }, p0, [x11, x12, LSL #2]\n" - "mov z30.d, z16.d\n" + "fmin z28.s, p3/M, z28.s, z17.s\n" "addvl x11, x11, #1\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" "st1w { z29.s }, p0, [x22]\n" - "mov z29.d, z16.d\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" "st1w { z28.s }, p0, [x22, x12, LSL #2]\n" - "mov z28.d, z16.d\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z0.s, z5.s\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" "add x21, x5, #0x1\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" "ld1w { z0.s }, p3/Z, [x8]\n" "add x6, x6, #0x1\n" "fmla z31.s, p3/M, z1.s, z6.s\n" @@ -528,4 +520,4 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 1bfe7eb09c..070270764c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include <cstddef> #include <cstdint> -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -100,511 +100,429 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" - "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x14, ALL, MUL #2\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "mov x13, #0x0\n" - "ldp x12, x11, [x19, #0x0]\n" - "cntw x10\n" - "ldp x9, x28, [x19, #0x10]\n" - "sub x27, XZR, x10\n" - "ld1w { z16.s }, p3/Z, [x16]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "cntw x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" - "cmp x10, %x[n_channels]\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" - "addvl x16, x16, #6\n" - "ldp x26, x25, [x15, #0x0]\n" - "ldp x24, x23, [x15, #0x10]\n" - "ldp x20, x19, [x15, #0x20]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #6\n" + "ldp x26, x25, [x14, #0x0]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" "ld1w { z5.s }, p2/Z, [x26, x13, LSL #2]\n" - "prfm pldl1keep, [x26, x14]\n" "ld1w { z6.s }, p2/Z, [x25, x13, LSL #2]\n" - "prfm pldl1keep, [x25, x14]\n" "ld1w { z7.s }, p2/Z, [x24, x13, LSL #2]\n" - "prfm pldl1keep, [x24, x14]\n" "ld1w { z8.s }, p2/Z, [x23, x13, LSL #2]\n" - "prfm pldl1keep, [x23, x14]\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" - "prfm pldl1keep, [x19, x14]\n" - "ldp x22, x21, [x15, #0x30]\n" - "ldp x20, x19, [x15, #0x40]\n" - "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n" - "prfm pldl1keep, [x22, x14]\n" - "ld1w { z12.s }, p2/Z, [x21, x13, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" - "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n" - "prfm pldl1keep, [x19, x14]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ldp x26, x25, [x14, #0x40]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z0.s, z5.s\n" - "ldr x21, [x15, #0x50]\n" - "whilelt p1.s, x10, %x[n_channels]\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0x58]\n" - "incw x27\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" - "ldr x20, [x15, #0x60]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" + "ldr x24, [x14, #0x50]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" + "ldr x23, [x14, #0x58]\n" + "incw x9\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" + "ldr x22, [x14, #0x60]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.s, p3/M, z1.s, z8.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x20, [x14, #0x70]\n" "fmla z28.s, p3/M, z1.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x16]\n" + "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0x68]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n" + "ldr x26, [x14, #0x80]\n" "fmla z28.s, p3/M, z2.s, z5.s\n" - "ldr x20, [x15, #0x70]\n" + "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.s, p3/M, z3.s, z5.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x24, [x14, #0x90]\n" "fmla z28.s, p3/M, z3.s, z6.s\n" - "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z9.s\n" - "ldr x19, [x15, #0x78]\n" + "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n" + "ldr x23, [x14, #0x98]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x26, [x15, #0x80]\n" + "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z7.s\n" - "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.s, p3/M, z0.s, z8.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.s, p3/M, z0.s, z14.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z8.s\n" - "ldr x25, [x15, #0x88]\n" + "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.s, p3/M, z1.s, z11.s\n" - "ldr x24, [x15, #0x90]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z13.s\n" "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z5.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z28.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x15, #0x98]\n" + "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.s, p3/M, z3.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.s, p3/M, z3.s, z6.s\n" - "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "ldr x20, [x15, #0xa0]\n" + "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z6.s\n" "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "addvl x16, x16, #16\n" + "ldr x22, [x14, #0xe0]\n" "fmla z28.s, p3/M, z4.s, z8.s\n" - "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z14.s\n" - "ldr x19, [x15, #0xa8]\n" + "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z11.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.s, p3/M, z0.s, z5.s\n" - "ldr x22, [x15, #0xb0]\n" "fmla z28.s, p3/M, z0.s, z6.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x22, x14]\n" "fmla z29.s, p3/M, z1.s, z6.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x15, #0xb8]\n" + "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.s, p3/M, z2.s, z9.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z10.s\n" - "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "ldr x20, [x15, #0xc0]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x19, [x15, #0xc8]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.s, p3/M, z4.s, z8.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z28.s, p3/M, z4.s, z14.s\n" - "ldr x21, [x15, #0xd0]\n" + "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z5.s\n" - "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0xd8]\n" "fmla z29.s, p3/M, z0.s, z9.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z1.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z5.s\n" - "ldr x20, [x15, #0xe0]\n" + "ld1w { z1.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0xe8]\n" "fmla z29.s, p3/M, z2.s, z5.s\n" - "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z6.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z3.s, z6.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z8.s\n" - "ldr x20, [x15, #0xf0]\n" + "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "ldr x19, [x15, #0xf8]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z14.s\n" - "ld1w { z1.s }, p3/Z, [x16]\n" "fmla z29.s, p3/M, z4.s, z8.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ldr x26, [x15, #0x100]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "ldr x25, [x15, #0x108]\n" "fmla z30.s, p3/M, z1.s, z5.s\n" - "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z5.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z5.s }, p1/Z, [x26, x12, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x24, [x15, #0x110]\n" - "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" + "incw x13\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "ldr x23, [x15, #0x118]\n" + "ldp x24, x23, [x14, #0x10]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z31.s, p3/M, z3.s, z6.s\n" + "ld1w { z6.s }, p1/Z, [x25, x12, LSL #2]\n" + "ldp x22, x21, [x14, #0x20]\n" "fmla z30.s, p3/M, z3.s, z8.s\n" - "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldp x20, x19, [x14, #0x30]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x24, x14]\n" - "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z7.s }, p1/Z, [x24, x12, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x23, x14]\n" - "incw x13\n" + "ld1w { z13.s }, p1/Z, [x21, x12, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z8.s\n" - "ldp x26, x25, [x15, #0x0]\n" - "whilelt p2.s, x13, %x[n_channels]\n" + "ld1w { z8.s }, p1/Z, [x23, x12, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "ldp x24, x23, [x15, #0x10]\n" - "addvl x14, x14, #1\n" + "ld1w { z11.s }, p1/Z, [x20, x12, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "ldp x20, x19, [x15, #0x20]\n" - "ldp x22, x21, [x15, #0x30]\n" + "ld1w { z12.s }, p1/Z, [x19, x12, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z9.s\n" - "ld1w { z5.s }, p1/Z, [x26, x10, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x22, x12, LSL #2]\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ldp x26, x25, [x14, #0x40]\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z6.s }, p1/Z, [x25, x10, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "prfm pldl1keep, [x25, x14]\n" + "addvl x15, x15, #16\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z7.s }, p1/Z, [x24, x10, LSL #2]\n" + "ld1w { z10.s }, p1/Z, [x26, x12, LSL #2]\n" + "ld1w { z14.s }, p1/Z, [x25, x12, LSL #2]\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "prfm pldl1keep, [x24, x14]\n" + "incw x12\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "ld1w { z8.s }, p1/Z, [x23, x10, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "prfm pldl1keep, [x23, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "ld1w { z9.s }, p1/Z, [x20, x10, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z13.s }, p1/Z, [x19, x10, LSL #2]\n" - "prfm pldl1keep, [x19, x14]\n" - "ld1w { z11.s }, p1/Z, [x22, x10, LSL #2]\n" - "prfm pldl1keep, [x22, x14]\n" - "ld1w { z12.s }, p1/Z, [x21, x10, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" - "ldp x20, x19, [x15, #0x40]\n" - "st1w { z31.s }, p0, [x12, x27, LSL #2]\n" - "st1w { z30.s }, p0, [x11, x27, LSL #2]\n" - "ld1w { z10.s }, p1/Z, [x20, x10, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z14.s }, p1/Z, [x19, x10, LSL #2]\n" - "incw x10\n" - "prfm pldl1keep, [x19, x14]\n" - "cmp x10, %x[n_channels]\n" - "st1w { z29.s }, p0, [x9, x27, LSL #2]\n" - "st1w { z28.s }, p0, [x28, x27, LSL #2]\n" - "ld1w { z16.s }, p3/Z, [x16, #4, MUL VL]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n" - "addvl x16, x16, #16\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n" - "addvl x16, x16, #-6\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z0.s, z5.s\n" - "ldr x21, [x15, #0x50]\n" - "incw x27\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0x58]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" + "ldr x24, [x14, #0x50]\n" + "incw x9\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" + "ldr x23, [x14, #0x58]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" - "ldr x20, [x15, #0x60]\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" + "ldr x22, [x14, #0x60]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.s, p3/M, z1.s, z8.s\n" "fmla z28.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z0.s }, p3/Z, [x16]\n" + "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n" + "ldr x20, [x14, #0x70]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0x68]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.s, p3/M, z2.s, z13.s\n" "fmla z28.s, p3/M, z2.s, z5.s\n" - "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n" - "ldr x20, [x15, #0x70]\n" + "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n" + "ldr x26, [x14, #0x80]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.s, p3/M, z3.s, z5.s\n" "fmla z28.s, p3/M, z3.s, z6.s\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z9.s\n" - "ldr x19, [x15, #0x78]\n" + "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z6.s\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n" - "ldr x26, [x15, #0x80]\n" + "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.s, p3/M, z0.s, z7.s\n" - "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.s, p3/M, z0.s, z8.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.s, p3/M, z0.s, z14.s\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "prfm pldl1keep, [x26, x14]\n" - "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z31.s, p3/M, z1.s, z8.s\n" - "ldr x25, [x15, #0x88]\n" + "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.s, p3/M, z1.s, z11.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z31.s, p3/M, z2.s, z13.s\n" "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z5.s\n" - "ldr x24, [x15, #0x90]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" "fmla z28.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x15, #0x98]\n" - "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.s, p3/M, z3.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.s, p3/M, z3.s, z6.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x23, x14]\n" - "ldr x20, [x15, #0xa0]\n" + "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z6.s\n" "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "addvl x16, x16, #16\n" "fmla z28.s, p3/M, z4.s, z8.s\n" - "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n" + "ldr x22, [x14, #0xe0]\n" "fmla z31.s, p3/M, z0.s, z14.s\n" - "ldr x19, [x15, #0xa8]\n" + "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z11.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.s, p3/M, z0.s, z5.s\n" - "ldr x22, [x15, #0xb0]\n" "fmla z28.s, p3/M, z0.s, z6.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x22, x14]\n" "fmla z29.s, p3/M, z1.s, z6.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x15, #0xb8]\n" + "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.s, p3/M, z2.s, z9.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z10.s\n" - "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "ldr x20, [x15, #0xc0]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x19, [x15, #0xc8]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.s, p3/M, z4.s, z8.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z28.s, p3/M, z4.s, z14.s\n" - "ldr x21, [x15, #0xd0]\n" + "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z5.s\n" - "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0xd8]\n" "fmla z29.s, p3/M, z0.s, z9.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z1.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z5.s\n" - "ldr x20, [x15, #0xe0]\n" + "ld1w { z1.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0xe8]\n" "fmla z29.s, p3/M, z2.s, z5.s\n" - "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z6.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z3.s, z6.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z8.s\n" - "ldr x20, [x15, #0xf0]\n" + "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "ldr x19, [x15, #0xf8]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z14.s\n" - "ld1w { z1.s }, p3/Z, [x16]\n" "fmla z29.s, p3/M, z4.s, z8.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ldr x26, [x15, #0x100]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "ldr x25, [x15, #0x108]\n" "fmla z30.s, p3/M, z1.s, z5.s\n" - "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z5.s\n" - "prfm pldl1keep, [x25, x14]\n" "fmla z30.s, p3/M, z2.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x24, [x15, #0x110]\n" - "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "ldr x23, [x15, #0x118]\n" "fmla z31.s, p3/M, z3.s, z6.s\n" "fmla z30.s, p3/M, z3.s, z8.s\n" - "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x24, x14]\n" - "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z31.s, p3/M, z4.s, z8.s\n" "fmla z30.s, p3/M, z4.s, z10.s\n" "fmla z29.s, p3/M, z4.s, z12.s\n" @@ -612,22 +530,22 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "fmax z31.s, p3/M, z31.s, z18.s\n" "fmax z30.s, p3/M, z30.s, z18.s\n" "fmax z29.s, p3/M, z29.s, z18.s\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x12, x27, LSL #2]\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x11, x27, LSL #2]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "st1w { z29.s }, p0, [x9, x27, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x28, x27, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index c501c67a5b..b524fd7c93 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -29,7 +29,7 @@ #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2) +#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -72,4 +72,4 @@ struct sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2) +#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 40220ad84e..52dc4683ad 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include <cstddef> #include <cstdint> diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 39ab3534f5..34ba8ece12 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include <cstddef> #include <cstdint> diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 7f4272672c..441da6da7a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include <cstddef> #include <cstdint> |