aboutsummaryrefslogtreecommitdiff
path: root/src/core/NEON/kernels/arm_conv/depthwise/kernels
diff options
context:
space:
mode:
authorFreddie Liardet <frederick.liardet@arm.com>2021-08-03 15:57:32 +0100
committerGeorgios Pinitas <georgios.pinitas@arm.com>2021-09-07 11:13:44 +0000
commitd216f570750b8ccde3754c4aef53fc20a90cb32d (patch)
tree83a88d3d4391c6a8ca5dabc73c763e6f0878c595 /src/core/NEON/kernels/arm_conv/depthwise/kernels
parent4e53c5ab47a713ab0ce53d076e2e4cf274fec312 (diff)
downloadComputeLibrary-d216f570750b8ccde3754c4aef53fc20a90cb32d.tar.gz
Update cpu depthwise kernels
Resolves: COMPMID-4688 Signed-off-by: Freddie Liardet <frederick.liardet@arm.com> Change-Id: I9e22f967f5b7ccaebff2fc49f0253f621d62d820 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6030 Tested-by: Arm Jenkins <bsgcomp@arm.com> Reviewed-by: Pablo Marquez Tello <pablo.tello@arm.com> Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com> Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Diffstat (limited to 'src/core/NEON/kernels/arm_conv/depthwise/kernels')
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp2
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp2
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp2
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp152
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp266
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp244
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp410
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp370
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp596
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp200
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp310
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp296
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp458
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp326
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp296
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp422
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp406
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp192
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp670
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp390
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp386
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp71
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp44
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp510
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp4
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp2
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp2
-rw-r--r--src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp2
67 files changed, 4399 insertions, 4061 deletions
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index bb43d57018..be50d1c535 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst
+class a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index 143070cc7e..a85e44360e 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -721,4 +721,4 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 35a9714944..a0a44997d1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -696,4 +696,4 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index 90db8703b5..39fa7f62fd 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst
+class a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index c2d67661f6..9b4b3ee50d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1156,4 +1156,4 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index fc988913be..e0abca91a2 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1289,4 +1289,4 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index df5328724d..1e0d922be4 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst
+class a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index c403576cf1..b5bee7ae7c 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1734,4 +1734,4 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 2e877285ae..e493104c03 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -2006,4 +2006,4 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index ca367cc1af..d89ae0cd6d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst
+class a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 9b2d77fa53..a5df51c4f9 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -893,4 +893,4 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index 4857077b05..61c58186f5 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -896,4 +896,4 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 53d2a3a8e1..6b5f91fa64 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst
+class a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@ struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index 027fd7f4c7..b08059db0a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1385,4 +1385,4 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 43eac0e6bc..5b086ec1ff 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1426,4 +1426,4 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index 88f20bb125..a888eb5776 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst
+class a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index fae208fbab..73c1e07420 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -522,3 +524,5 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 2f93a68c23..42931fba17 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -509,3 +511,5 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index 6a882ec52f..01bb06a561 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst
+class a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index 401528aa59..6d185e7274 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -823,3 +825,5 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index 39ec001ae1..aa7d35e3e1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -901,3 +903,5 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index 84bac12429..17084b57d5 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst
+class a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index 616fd0d0e7..6faacf144a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -1227,3 +1229,5 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 51a5679bff..aeaf1049f1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -1393,3 +1395,5 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index 8eb560562b..f23862b699 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst
+class a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 4466ec1974..65e487ddbb 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -610,3 +612,5 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index a5153019e7..4b24862eb2 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -625,3 +627,5 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 314fe766de..e4bfbe6783 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if defined(__aarch64__)
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst
+class a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@ struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__aarch64__)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index b74a3c9b7c..8b030ecc8b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if defined(__aarch64__)
+
namespace arm_conv {
namespace depthwise {
@@ -986,3 +988,5 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__aarch64__)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index adc5170ef7..4754a6f6f1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if defined(__aarch64__)
+
namespace arm_conv {
namespace depthwise {
@@ -1039,3 +1041,5 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__aarch64__)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
index 14e113b776..b62ebb1687 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
index ccdde41973..8d22836e5b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
index 699cc6c80c..b42f29ab0d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index b8e59306d5..1cfea9dcbd 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst
+class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index a4c1a40100..955a02de57 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,149 +113,141 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
"add x27, x10, x22, LSL #1\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x26, x27, x22, LSL #1\n"
- "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x14]\n"
"add x25, x26, x22, LSL #1\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n"
"add x24, x11, x11\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n"
"add x23, x24, x11\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n"
"mul x19, x17, x20\n" // offset = tile_i * ld_output_row
- "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n"
"madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col
- "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n"
"mul x19, x19, x15\n" // offset *= output_tile_size
- "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n"
"add x28, x28, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
- "whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z10.h }, p2/Z, [x10]\n" // Load input point (0, 0)
+ "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n"
"add x22, x28, x20, LSL #1\n"
- "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3)
+ "whilelt p2.h, XZR, %x[n_channels]\n"
+ "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x10]\n"
"addvl x14, x14, #16\n"
- "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n"
"cmp x12, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n"
"addvl x14, x14, #-6\n"
- "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z4.h, z9.h\n"
- "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
"whilelt p1.h, x12, %x[n_channels]\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
"inch x21\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0)
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x25]\n"
"inch x13\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n"
"inch x12\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
+ "ld1h { z16.h }, p3/Z, [x14]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z31.h, p3/M, z5.h, z12.h\n"
"fmla z30.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n"
"addvl x10, x10, #1\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n"
"addvl x27, x27, #1\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0)
- "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z9.h }, p2/Z, [x26]\n"
+ "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
- "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n"
"addvl x26, x26, #1\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
- "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n"
"whilelt p2.h, x13, %x[n_channels]\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n"
"addvl x25, x25, #1\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n"
"cmp x12, %x[n_channels]\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x10]\n" // Load input point (0, 0)
+ "ld1h { z10.h }, p1/Z, [x10]\n"
"fmla z28.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3)
- "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n"
+ "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
"addvl x14, x14, #16\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n"
"addvl x14, x14, #-6\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0)
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x28]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z16.d\n"
- "addvl x28, x28, #1\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "mov z29.d, z16.d\n"
+ "st1h { z30.h }, p0, [x28, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
+ "addvl x28, x28, #1\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1)
- "mov z28.d, z16.d\n"
+ "st1h { z29.h }, p0, [x22]\n"
+ "st1h { z28.h }, p0, [x22, x9, LSL #1]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z4.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
"ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
"ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x17, #0x1\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0)
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x25]\n"
"add x16, x16, #0x1\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"cmp x16, x19\n"
@@ -263,10 +255,10 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
"fmla z30.h, p3/M, z4.h, z12.h\n"
"csel x16, x16, XZR, LT\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n"
"csel x17, x17, x21, LT\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n"
"cmp x17, x20\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
@@ -274,24 +266,24 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0)
+ "ld1h { z9.h }, p2/Z, [x26]\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
@@ -303,14 +295,14 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x28]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x28, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
+ "st1h { z29.h }, p0, [x22]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z28.h }, p0, [x22, x9, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
@@ -321,4 +313,4 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index a845e7c0c6..7cca6fbcbf 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -54,22 +54,22 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
const __fp16 max
) : outptrs(outptrs), params(params), min(min), max(max)
{
- inptrs[0] = input_ptrs[0];
- inptrs[1] = input_ptrs[1];
- inptrs[2] = input_ptrs[2];
- inptrs[3] = input_ptrs[3];
- inptrs[4] = input_ptrs[4];
- inptrs[5] = input_ptrs[5];
- inptrs[6] = input_ptrs[6];
- inptrs[7] = input_ptrs[7];
- inptrs[8] = input_ptrs[8];
- inptrs[9] = input_ptrs[9];
- inptrs[10] = input_ptrs[10];
- inptrs[11] = input_ptrs[11];
- inptrs[12] = input_ptrs[12];
- inptrs[13] = input_ptrs[13];
- inptrs[14] = input_ptrs[14];
- inptrs[15] = input_ptrs[15];
+ inptrs[0] = input_ptrs[5];
+ inptrs[1] = input_ptrs[0];
+ inptrs[2] = input_ptrs[3];
+ inptrs[3] = input_ptrs[6];
+ inptrs[4] = input_ptrs[9];
+ inptrs[5] = input_ptrs[12];
+ inptrs[6] = input_ptrs[15];
+ inptrs[7] = input_ptrs[1];
+ inptrs[8] = input_ptrs[2];
+ inptrs[9] = input_ptrs[10];
+ inptrs[10] = input_ptrs[4];
+ inptrs[11] = input_ptrs[7];
+ inptrs[12] = input_ptrs[8];
+ inptrs[13] = input_ptrs[11];
+ inptrs[14] = input_ptrs[13];
+ inptrs[15] = input_ptrs[14];
}
};
@@ -78,207 +78,219 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x3, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x4, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x5, #0x0\n"
- "ldp x6, x7, [x19, #0x0]\n"
- "cnth x8\n"
- "ldp x17, x16, [x19, #0x10]\n"
- "sub x15, XZR, x8\n"
- "ldp x14, x13, [x19, #0x20]\n"
- "whilelt p2.h, XZR, %x[n_channels]\n"
- "ldp x12, x11, [x19, #0x30]\n"
- "cmp x8, %x[n_channels]\n"
- "ldp x10, x9, [x19, #0x40]\n"
- "ldp x28, x27, [x19, #0x50]\n"
- "ldp x26, x25, [x19, #0x60]\n"
- "ldp x24, x23, [x19, #0x70]\n"
- "ldp x22, x21, [x3, #0x0]\n"
- "ldp x20, x19, [x3, #0x10]\n"
+ "mov x13, #0x0\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
- "addvl x4, x4, #16\n"
- "ld1h { z9.h }, p2/Z, [x13, x5, LSL #1]\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x4, x4, #-6\n"
- "ld1h { z10.h }, p2/Z, [x6, x5, LSL #1]\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x9, x5, LSL #1]\n"
+ "cnth x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
+ "whilelt p2.h, XZR, %x[n_channels]\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldr x22, [x14, #0x20]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z4.h, z9.h\n"
- "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias
- "whilelt p1.h, x8, %x[n_channels]\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
- "inch x15\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
+ "ldr x21, [x14, #0x28]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
+ "ldr x20, [x14, #0x30]\n"
+ "inch x9\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
+ "ldr x19, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z12.h\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x9, x8, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
+ "ldr x22, [x14, #0x20]\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
+ "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n"
- "inch x5\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
- "whilelt p2.h, x5, %x[n_channels]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x13, x8, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n"
"fmla z28.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x16, x8, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x6, x8, LSL #1]\n"
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
- "addvl x4, x4, #16\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x12, x8, LSL #1]\n"
- "inch x8\n"
+ "addvl x15, x15, #16\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x8, %x[n_channels]\n"
+ "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x4, x4, #-6\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
- "mov z30.d, z16.d\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
- "mov z29.d, z16.d\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
- "mov z28.d, z16.d\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z4.h, z9.h\n"
- "inch x15\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
+ "ldr x21, [x14, #0x28]\n"
+ "inch x9\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
+ "ldr x20, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
+ "ldr x19, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z31.h, p3/M, z5.h, z12.h\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n"
- "fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
+ "fmla z30.h, p3/M, z5.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
- "fmla z30.h, p3/M, z8.h, z10.h\n"
"fmla z28.h, p3/M, z6.h, z11.h\n"
+ "fmla z30.h, p3/M, z8.h, z10.h\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
+ "fmla z28.h, p3/M, z7.h, z12.h\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "fmla z28.h, p3/M, z7.h, z12.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index e1f23aae66..af8af1840a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst
+class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index 0708f578a8..4126cefa34 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,107 +113,98 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"add x11, x14, x22, LSL #1\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x10, x11, x22, LSL #1\n"
- "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x8]\n"
"add x9, x10, x22, LSL #1\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n"
"add x28, x9, x22, LSL #1\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n"
"add x27, x15, x15\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n"
"add x26, x27, x15\n"
- "mov z27.d, z16.d\n"
- "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n"
"add x25, x26, x15\n"
- "mov z26.d, z16.d\n"
- "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n"
"mul x19, x6, x20\n" // offset = tile_i * ld_output_row
- "mov z25.d, z16.d\n"
- "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n"
"madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col
- "mov z24.d, z16.d\n"
+ "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n"
"mul x19, x19, x23\n" // offset *= output_tile_size
- "mov z23.d, z16.d\n"
- "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
"add x24, x13, x13\n"
+ "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
"add x23, x12, x20, LSL #1\n"
"add x22, x23, x20, LSL #1\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2)
- "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (0, 0)
+ "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x14]\n"
"addvl x8, x8, #16\n"
- "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n"
"cmp x16, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n"
"addvl x8, x8, #-6\n"
- "ld1h { z12.h }, p2/Z, [x28]\n" // Load input point (4, 0)
- "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z12.h }, p2/Z, [x28]\n"
+ "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
"whilelt p1.h, x16, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
"inch x21\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
"inch x17\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
"inch x16\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z16.h }, p3/Z, [x8]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x11]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0)
+ "ld1h { z12.h }, p2/Z, [x9]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
@@ -223,173 +214,164 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n"
"addvl x11, x11, #1\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
"fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n"
"addvl x14, x14, #1\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
- "ld1h { z10.h }, p1/Z, [x14]\n" // Load input point (0, 0)
+ "ld1h { z10.h }, p1/Z, [x14]\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n"
"addvl x9, x9, #1\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z12.h }, p2/Z, [x10]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n"
"addvl x10, x10, #1\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
- "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n"
"whilelt p2.h, x17, %x[n_channels]\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n"
"addvl x28, x28, #1\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n"
"cmp x16, %x[n_channels]\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x28]\n" // Load input point (4, 0)
+ "ld1h { z12.h }, p1/Z, [x28]\n"
"fmla z29.h, p3/M, z8.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z26.h, p3/M, z5.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z23.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n"
"fmla z24.h, p3/M, z7.h, z13.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z25.h, p3/M, z8.h, z13.h\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
"fmla z23.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x8, x8, #16\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n"
"addvl x8, x8, #-6\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0)
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x12]\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z16.d\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2)
- "mov z29.d, z16.d\n"
- "addvl x12, x12, #1\n"
+ "st1h { z30.h }, p0, [x12, x13, LSL #1]\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
+ "st1h { z29.h }, p0, [x12, x24, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0)
- "mov z28.d, z16.d\n"
+ "addvl x12, x12, #1\n"
+ "fmax z25.h, p3/M, z25.h, z18.h\n"
+ "st1h { z28.h }, p0, [x23]\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
- "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1)
- "mov z27.d, z16.d\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2)
- "mov z26.d, z16.d\n"
- "addvl x23, x23, #1\n"
- "fmax z25.h, p3/M, z25.h, z18.h\n"
+ "st1h { z27.h }, p0, [x23, x13, LSL #1]\n"
+ "fmin z25.h, p3/M, z25.h, z17.h\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
+ "st1h { z26.h }, p0, [x23, x24, LSL #1]\n"
+ "addvl x23, x23, #1\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
- "fmin z25.h, p3/M, z25.h, z17.h\n"
- "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0)
- "mov z25.d, z16.d\n"
+ "st1h { z25.h }, p0, [x22]\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
- "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1)
- "mov z24.d, z16.d\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2)
- "mov z23.d, z16.d\n"
+ "st1h { z24.h }, p0, [x22, x13, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x24, LSL #1]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
"ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
"ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x6, #0x1\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x7, x7, #0x1\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
"cmp x7, x19\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
"csel x7, x7, XZR, LT\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
"csel x6, x6, x21, LT\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
"cmp x6, x20\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x11]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0)
+ "ld1h { z12.h }, p2/Z, [x9]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
@@ -399,36 +381,36 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
"fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z12.h }, p2/Z, [x10]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
@@ -442,29 +424,29 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"fmax z30.h, p3/M, z30.h, z18.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x12]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x12, x13, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
- "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2)
+ "st1h { z29.h }, p0, [x12, x24, LSL #1]\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
"fmax z25.h, p3/M, z25.h, z18.h\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0)
+ "st1h { z28.h }, p0, [x23]\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z27.h }, p0, [x23, x13, LSL #1]\n"
"fmin z25.h, p3/M, z25.h, z17.h\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
- "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2)
+ "st1h { z26.h }, p0, [x23, x24, LSL #1]\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
- "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0)
+ "st1h { z25.h }, p0, [x22]\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1)
- "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2)
+ "st1h { z24.h }, p0, [x22, x13, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x24, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
@@ -475,4 +457,4 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index 770576c5da..f79a36b2a3 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -87,130 +87,121 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x6, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x7, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x8, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x17, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x16\n"
- "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias
- "sub x15, XZR, x16\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias
+ "cnth x12\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
+ "sub x11, XZR, x12\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias
- "cmp x16, %x[n_channels]\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias
- "mov z27.d, z16.d\n"
- "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias
- "mov z26.d, z16.d\n"
- "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias
- "mov z25.d, z16.d\n"
- "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #16\n"
- "mov z24.d, z16.d\n"
- "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias
- "mov z23.d, z16.d\n"
- "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #-6\n"
- "ldp x14, x13, [x8, #0x0]\n"
- "ldp x12, x11, [x8, #0x10]\n"
- "ldr x10, [x8, #0x20]\n"
- "ld1h { z9.h }, p2/Z, [x14, x17, LSL #1]\n"
- "ld1h { z10.h }, p2/Z, [x13, x17, LSL #1]\n"
- "ld1h { z11.h }, p2/Z, [x12, x17, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x10, x17, LSL #1]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ldr x26, [x14, #0x20]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x9, [x8, #0x28]\n"
- "whilelt p1.h, x16, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x28, [x8, #0x30]\n"
- "inch x15\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x27, [x8, #0x38]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
+ "inch x11\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
- "ldr x26, [x8, #0x40]\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
- "ldr x22, [x8, #0x48]\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "ldr x21, [x8, #0x50]\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
- "ldr x20, [x8, #0x58]\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
- "ldr x19, [x8, #0x60]\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
- "ldr x25, [x8, #0x68]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
- "ldr x24, [x8, #0x70]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x23, [x8, #0x78]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
- "ldr x14, [x8, #0x80]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
- "ldr x13, [x8, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
- "ldr x12, [x8, #0x90]\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
- "ldr x11, [x8, #0x98]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
- "ldr x10, [x8, #0xa0]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
- "ldr x9, [x8, #0xa8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
- "ldr x28, [x8, #0xb0]\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
- "ldr x27, [x8, #0xb8]\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
- "ldr x26, [x8, #0xc0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ldr x22, [x6, #0x0]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
- "ldr x21, [x6, #0x8]\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
- "ldr x20, [x6, #0x10]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
- "ldr x19, [x6, #0x18]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
- "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z8.h, z10.h\n"
@@ -219,191 +210,182 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ldp x14, x13, [x8, #0x0]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
- "ld1h { z9.h }, p1/Z, [x14, x16, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n"
- "fmla z30.h, p3/M, z5.h, z11.h\n"
- "ld1h { z10.h }, p1/Z, [x13, x16, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ldp x12, x11, [x8, #0x10]\n"
+ "fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
- "ldr x10, [x8, #0x20]\n"
+ "ldr x26, [x14, #0x20]\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
- "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n"
- "inch x17\n"
+ "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
- "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias
- "whilelt p2.h, x17, %x[n_channels]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias
+ "ldp x28, x27, [x14, #0x10]\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x11, x16, LSL #1]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z29.h, p3/M, z8.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n"
"fmla z26.h, p3/M, z5.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n"
"fmla z23.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x12, x16, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n"
"fmla z25.h, p3/M, z8.h, z13.h\n"
- "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n"
"fmla z24.h, p3/M, z7.h, z13.h\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z23.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x10, x16, LSL #1]\n"
- "inch x16\n"
+ "ld1h { z13.h }, p1/Z, [x26, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #16\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x16, %x[n_channels]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #-6\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
- "mov z31.d, z16.d\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "fmax z27.h, p3/M, z27.h, z18.h\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "ldr x22, [x6, #0x20]\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
- "mov z30.d, z16.d\n"
- "fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
- "mov z29.d, z16.d\n"
- "ldr x21, [x6, #0x28]\n"
- "fmax z27.h, p3/M, z27.h, z18.h\n"
- "ldr x20, [x6, #0x30]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
- "mov z28.d, z16.d\n"
- "ldr x19, [x6, #0x38]\n"
- "fmax z25.h, p3/M, z25.h, z18.h\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
+ "fmin z28.h, p3/M, z28.h, z17.h\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
- "st1h { z27.h }, p0, [x22, x15, LSL #1]\n"
- "mov z27.d, z16.d\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "ldr x22, [x6, #0x40]\n"
- "fmin z25.h, p3/M, z25.h, z17.h\n"
- "st1h { z26.h }, p0, [x21, x15, LSL #1]\n"
- "mov z26.d, z16.d\n"
+ "ldr x21, [x16, #0x28]\n"
+ "fmax z25.h, p3/M, z25.h, z18.h\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
- "st1h { z25.h }, p0, [x20, x15, LSL #1]\n"
- "mov z25.d, z16.d\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
+ "fmin z25.h, p3/M, z25.h, z17.h\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
- "st1h { z24.h }, p0, [x19, x15, LSL #1]\n"
- "mov z24.d, z16.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z23.h }, p0, [x22, x15, LSL #1]\n"
- "mov z23.d, z16.d\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x9, [x8, #0x28]\n"
- "inch x15\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x28, [x8, #0x30]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
+ "inch x11\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x27, [x8, #0x38]\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
- "ldr x26, [x8, #0x40]\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
- "ldr x22, [x8, #0x48]\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "ldr x21, [x8, #0x50]\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
- "ldr x20, [x8, #0x58]\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
- "ldr x19, [x8, #0x60]\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
- "ldr x25, [x8, #0x68]\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
- "ldr x24, [x8, #0x70]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x23, [x8, #0x78]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
- "ldr x14, [x8, #0x80]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
- "ldr x13, [x8, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
- "ldr x12, [x8, #0x90]\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
- "ldr x11, [x8, #0x98]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
- "ldr x10, [x8, #0xa0]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
- "ldr x9, [x8, #0xa8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
- "ldr x28, [x8, #0xb0]\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
- "ldr x27, [x8, #0xb8]\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
- "ldr x26, [x8, #0xc0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ldr x22, [x6, #0x0]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
- "ldr x21, [x6, #0x8]\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
- "ldr x20, [x6, #0x10]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
- "ldr x19, [x6, #0x18]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z8.h, z10.h\n"
@@ -412,36 +394,36 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
"fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
@@ -455,41 +437,41 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
"fmax z30.h, p3/M, z30.h, z18.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "ldr x22, [x6, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "ldr x21, [x6, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z25.h, p3/M, z25.h, z18.h\n"
- "ldr x20, [x6, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "ldr x19, [x6, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z25.h, p3/M, z25.h, z17.h\n"
- "st1h { z27.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
- "st1h { z26.h }, p0, [x21, x15, LSL #1]\n"
- "st1h { z25.h }, p0, [x20, x15, LSL #1]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z24.h }, p0, [x19, x15, LSL #1]\n"
- "ldr x22, [x6, #0x40]\n"
- "st1h { z23.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
+ "ldr x22, [x16, #0x40]\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index 7d035f0571..60234c8a86 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst
+class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index 93e1908df7..3cfac06449 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,92 +113,75 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"add x15, x8, x22, LSL #1\n"
"ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x14, x15, x22, LSL #1\n"
- "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias
- "mov z31.d, z13.d\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z13.h }, p3/Z, [x4]\n"
"add x13, x14, x22, LSL #1\n"
- "mov z30.d, z13.d\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n"
"add x12, x13, x22, LSL #1\n"
- "mov z29.d, z13.d\n"
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n"
"add x11, x12, x22, LSL #1\n"
- "mov z28.d, z13.d\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n"
"add x10, x7, x7\n"
- "mov z27.d, z13.d\n"
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n"
"add x9, x10, x7\n"
- "mov z26.d, z13.d\n"
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n"
"add x28, x9, x7\n"
- "mov z25.d, z13.d\n"
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n"
"add x27, x28, x7\n"
- "mov z24.d, z13.d\n"
+ "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n"
"mul x19, x2, x20\n" // offset = tile_i * ld_output_row
- "mov z23.d, z13.d\n"
+ "add x26, x17, x17\n"
"madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col
- "mov z22.d, z13.d\n"
"mul x19, x19, x23\n" // offset *= output_tile_size
- "mov z21.d, z13.d\n"
"add x16, x16, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
- "mov z20.d, z13.d\n"
- "add x26, x16, x20, LSL #1\n"
- "mov z19.d, z13.d\n"
- "add x25, x26, x20, LSL #1\n"
- "mov z18.d, z13.d\n"
+ "add x25, x16, x20, LSL #1\n"
"add x24, x25, x20, LSL #1\n"
- "mov z17.d, z13.d\n"
- "add x23, x17, x17\n"
- "mov z16.d, z13.d\n"
- "add x22, x23, x17\n"
+ "add x23, x24, x20, LSL #1\n"
+ "add x22, x26, x17\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2)
- "ld1h { z10.h }, p2/Z, [x8]\n" // Load input point (0, 0)
+ "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x8]\n"
"addvl x4, x4, #16\n"
- "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n"
"cmp x6, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n"
"addvl x4, x4, #-6\n"
- "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
"whilelt p1.h, x6, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
"inch x21\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
"inch x5\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
"inch x6\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2)
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0)
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z10.h }, p2/Z, [x11]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1)
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3)
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4)
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
@@ -206,15 +189,16 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmla z22.h, p3/M, z4.h, z9.h\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0)
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
+ "ld1h { z13.h }, p3/Z, [x4]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5)
+ "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0)
+ "ld1h { z11.h }, p2/Z, [x12]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
@@ -224,261 +208,245 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n"
"addvl x8, x8, #1\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0)
+ "ld1h { z10.h }, p2/Z, [x14]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n"
"addvl x14, x14, #1\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
- "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0)
+ "ld1h { z10.h }, p2/Z, [x13]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n"
"addvl x13, x13, #1\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n"
"addvl x11, x11, #1\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n"
"addvl x15, x15, #1\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n"
"whilelt p2.h, x5, %x[n_channels]\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n"
"addvl x12, x12, #1\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"cmp x6, %x[n_channels]\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n"
"fmla z24.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n"
"fmla z23.h, p3/M, z7.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n"
"fmla z22.h, p3/M, z6.h, z12.h\n"
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n"
"fmla z19.h, p3/M, z4.h, z12.h\n"
"fmla z18.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n"
"fmla z21.h, p3/M, z8.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n"
"fmla z20.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z5.h, z10.h\n"
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n"
"fmla z16.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x8]\n" // Load input point (0, 0)
+ "ld1h { z10.h }, p1/Z, [x8]\n"
"fmax z31.h, p3/M, z31.h, z15.h\n"
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n"
"addvl x4, x4, #16\n"
"fmax z30.h, p3/M, z30.h, z15.h\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z15.h\n"
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n"
"addvl x4, x4, #-6\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0)
- "mov z31.d, z13.d\n"
+ "st1h { z31.h }, p0, [x16]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
- "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z13.d\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2)
- "mov z29.d, z13.d\n"
+ "st1h { z30.h }, p0, [x16, x17, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
+ "st1h { z29.h }, p0, [x16, x26, LSL #1]\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
+ "fmax z24.h, p3/M, z24.h, z15.h\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
- "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3)
- "mov z28.d, z13.d\n"
- "addvl x16, x16, #1\n"
+ "st1h { z28.h }, p0, [x16, x22, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
- "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0)
- "mov z27.d, z13.d\n"
+ "addvl x16, x16, #1\n"
"fmin z26.h, p3/M, z26.h, z14.h\n"
- "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1)
- "mov z26.d, z13.d\n"
+ "st1h { z27.h }, p0, [x25]\n"
"fmin z25.h, p3/M, z25.h, z14.h\n"
- "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2)
- "mov z25.d, z13.d\n"
- "fmax z24.h, p3/M, z24.h, z15.h\n"
+ "fmin z24.h, p3/M, z24.h, z14.h\n"
+ "st1h { z26.h }, p0, [x25, x17, LSL #1]\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
+ "st1h { z25.h }, p0, [x25, x26, LSL #1]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
- "fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3)
- "mov z24.d, z13.d\n"
- "addvl x26, x26, #1\n"
+ "st1h { z24.h }, p0, [x25, x22, LSL #1]\n"
+ "addvl x25, x25, #1\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
- "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0)
- "mov z23.d, z13.d\n"
+ "st1h { z23.h }, p0, [x24]\n"
"fmin z22.h, p3/M, z22.h, z14.h\n"
- "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1)
- "mov z22.d, z13.d\n"
"fmin z21.h, p3/M, z21.h, z14.h\n"
- "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2)
- "mov z21.d, z13.d\n"
+ "st1h { z22.h }, p0, [x24, x17, LSL #1]\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
+ "st1h { z21.h }, p0, [x24, x26, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
+ "fmax z16.h, p3/M, z16.h, z15.h\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3)
- "mov z20.d, z13.d\n"
- "addvl x25, x25, #1\n"
+ "st1h { z20.h }, p0, [x24, x22, LSL #1]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0)
- "mov z19.d, z13.d\n"
+ "addvl x24, x24, #1\n"
"fmin z18.h, p3/M, z18.h, z14.h\n"
- "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1)
- "mov z18.d, z13.d\n"
+ "st1h { z19.h }, p0, [x23]\n"
"fmin z17.h, p3/M, z17.h, z14.h\n"
- "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2)
- "mov z17.d, z13.d\n"
- "fmax z16.h, p3/M, z16.h, z15.h\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3)
- "mov z16.d, z13.d\n"
- "addvl x24, x24, #1\n"
+ "st1h { z18.h }, p0, [x23, x17, LSL #1]\n"
+ "st1h { z17.h }, p0, [x23, x26, LSL #1]\n"
+ "st1h { z16.h }, p0, [x23, x22, LSL #1]\n"
+ "addvl x23, x23, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
"ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
"ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x2, #0x1\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x3, x3, #0x1\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
"cmp x3, x19\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
"csel x3, x3, XZR, LT\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
"csel x2, x2, x21, LT\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2)
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n"
"cmp x2, x20\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0)
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z10.h }, p2/Z, [x11]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1)
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3)
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4)
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
@@ -486,15 +454,15 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmla z22.h, p3/M, z4.h, z9.h\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0)
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5)
+ "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0)
+ "ld1h { z11.h }, p2/Z, [x12]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
@@ -504,117 +472,117 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0)
+ "ld1h { z10.h }, p2/Z, [x14]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0)
+ "ld1h { z10.h }, p2/Z, [x13]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
@@ -632,49 +600,49 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmax z29.h, p3/M, z29.h, z15.h\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x16]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x16, x17, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
- "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2)
+ "st1h { z29.h }, p0, [x16, x26, LSL #1]\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
- "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3)
+ "st1h { z28.h }, p0, [x16, x22, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0)
+ "st1h { z27.h }, p0, [x25]\n"
"fmin z26.h, p3/M, z26.h, z14.h\n"
"fmin z25.h, p3/M, z25.h, z14.h\n"
- "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z26.h }, p0, [x25, x17, LSL #1]\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
- "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2)
+ "st1h { z25.h }, p0, [x25, x26, LSL #1]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
"fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3)
+ "st1h { z24.h }, p0, [x25, x22, LSL #1]\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
"fmin z22.h, p3/M, z22.h, z14.h\n"
- "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0)
+ "st1h { z23.h }, p0, [x24]\n"
"fmin z21.h, p3/M, z21.h, z14.h\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1)
+ "st1h { z22.h }, p0, [x24, x17, LSL #1]\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
- "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2)
+ "st1h { z21.h }, p0, [x24, x26, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
- "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3)
+ "st1h { z20.h }, p0, [x24, x22, LSL #1]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0)
+ "st1h { z19.h }, p0, [x23]\n"
"fmin z18.h, p3/M, z18.h, z14.h\n"
"fmin z17.h, p3/M, z17.h, z14.h\n"
- "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1)
+ "st1h { z18.h }, p0, [x23, x17, LSL #1]\n"
"fmax z16.h, p3/M, z16.h, z15.h\n"
- "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2)
+ "st1h { z17.h }, p0, [x23, x26, LSL #1]\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3)
+ "st1h { z16.h }, p0, [x23, x22, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
@@ -685,4 +653,4 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 8eaf0a46d6..66f6c3bb7a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -98,570 +98,538 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x4, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z15.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x5, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x6\n"
- "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias
- "mov z31.d, z13.d\n"
- "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias
- "sub x7, XZR, x6\n"
- "mov z30.d, z13.d\n"
- "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias
+ "cnth x12\n"
+ "ld1h { z13.h }, p3/Z, [x15]\n"
+ "sub x11, XZR, x12\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "mov z29.d, z13.d\n"
- "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias
- "cmp x6, %x[n_channels]\n"
- "mov z28.d, z13.d\n"
- "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias
- "mov z27.d, z13.d\n"
- "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias
- "mov z26.d, z13.d\n"
- "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias
- "mov z25.d, z13.d\n"
- "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #16\n"
- "mov z24.d, z13.d\n"
- "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias
- "mov z23.d, z13.d\n"
- "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #-6\n"
- "mov z22.d, z13.d\n"
- "ldp x8, x17, [x4, #0x0]\n"
- "mov z21.d, z13.d\n"
- "ldp x16, x15, [x4, #0x10]\n"
- "mov z20.d, z13.d\n"
- "ld1h { z9.h }, p2/Z, [x8, x5, LSL #1]\n"
- "mov z19.d, z13.d\n"
- "mov z18.d, z13.d\n"
- "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n"
- "mov z17.d, z13.d\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
- "mov z16.d, z13.d\n"
- "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0x20]\n"
- "whilelt p1.h, x6, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0x28]\n"
- "inch x7\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0x30]\n"
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x20]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
+ "inch x11\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0x38]\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0x40]\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0x48]\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
- "ldr x28, [x4, #0x50]\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
- "ldr x27, [x4, #0x58]\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
- "ldr x26, [x4, #0x60]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
- "ldr x25, [x4, #0x68]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
- "ldr x24, [x4, #0x70]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ldr x23, [x4, #0x78]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "ldr x8, [x4, #0x80]\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
- "ldr x17, [x4, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
- "ldr x16, [x4, #0x90]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "ldr x15, [x4, #0x98]\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n"
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0xa0]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0xa8]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0xb0]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0xb8]\n"
"fmla z22.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0xc0]\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0xc8]\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "ldr x22, [x2, #0x0]\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x2, #0x8]\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
- "ldr x28, [x4, #0xd0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ldr x27, [x4, #0xd8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
- "ldr x26, [x4, #0xe0]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
- "ldr x20, [x2, #0x10]\n"
+ "ld1h { z13.h }, p3/Z, [x15]\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
- "ldr x19, [x2, #0x18]\n"
"fmla z22.h, p3/M, z5.h, z10.h\n"
- "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias
"fmla z21.h, p3/M, z4.h, z10.h\n"
"fmla z20.h, p3/M, z3.h, z10.h\n"
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ldr x25, [x4, #0xe8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
- "ldr x23, [x4, #0xf8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
- "ldr x24, [x4, #0xf0]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
- "ldr x17, [x4, #0x108]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ldr x8, [x4, #0x100]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
- "ldr x15, [x4, #0x118]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
- "ldr x16, [x4, #0x110]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
- "ldp x8, x17, [x4, #0x0]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
- "ld1h { z9.h }, p1/Z, [x8, x6, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n"
- "inch x5\n"
+ "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
- "ldp x16, x15, [x4, #0x10]\n"
- "whilelt p2.h, x5, %x[n_channels]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z24.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x16, x6, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n"
"fmla z23.h, p3/M, z7.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z22.h, p3/M, z6.h, z12.h\n"
- "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z19.h, p3/M, z4.h, z12.h\n"
"fmla z18.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x15, x6, LSL #1]\n"
+ "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n"
"fmla z21.h, p3/M, z8.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z20.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z5.h, z10.h\n"
- "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z16.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x17, x6, LSL #1]\n"
- "inch x6\n"
+ "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z31.h, p3/M, z31.h, z15.h\n"
- "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #16\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z30.h, p3/M, z30.h, z15.h\n"
- "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x6, %x[n_channels]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.h, p3/M, z29.h, z15.h\n"
- "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #-6\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x22, x7, LSL #1]\n"
- "mov z31.d, z13.d\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
- "ldr x22, [x2, #0x20]\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "st1h { z30.h }, p0, [x21, x7, LSL #1]\n"
- "mov z30.d, z13.d\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
- "st1h { z29.h }, p0, [x20, x7, LSL #1]\n"
- "mov z29.d, z13.d\n"
- "ldr x21, [x2, #0x28]\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
- "ldr x20, [x2, #0x30]\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
- "st1h { z28.h }, p0, [x19, x7, LSL #1]\n"
- "mov z28.d, z13.d\n"
- "ldr x19, [x2, #0x38]\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "st1h { z27.h }, p0, [x22, x7, LSL #1]\n"
- "mov z27.d, z13.d\n"
- "ldr x22, [x2, #0x40]\n"
- "fmin z26.h, p3/M, z26.h, z14.h\n"
- "st1h { z26.h }, p0, [x21, x7, LSL #1]\n"
- "mov z26.d, z13.d\n"
- "fmin z25.h, p3/M, z25.h, z14.h\n"
- "ldr x21, [x2, #0x48]\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
- "st1h { z25.h }, p0, [x20, x7, LSL #1]\n"
- "mov z25.d, z13.d\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
- "ldr x20, [x2, #0x50]\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
+ "fmin z26.h, p3/M, z26.h, z14.h\n"
+ "ldr x20, [x16, #0x30]\n"
+ "fmin z25.h, p3/M, z25.h, z14.h\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z24.h }, p0, [x19, x7, LSL #1]\n"
- "mov z24.d, z13.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
- "ldr x19, [x2, #0x58]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
- "st1h { z23.h }, p0, [x22, x7, LSL #1]\n"
- "mov z23.d, z13.d\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
- "ldr x22, [x2, #0x60]\n"
- "fmin z22.h, p3/M, z22.h, z14.h\n"
- "st1h { z22.h }, p0, [x21, x7, LSL #1]\n"
- "mov z22.d, z13.d\n"
- "fmin z21.h, p3/M, z21.h, z14.h\n"
- "ldr x21, [x2, #0x68]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
- "st1h { z21.h }, p0, [x20, x7, LSL #1]\n"
- "mov z21.d, z13.d\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
- "ldr x20, [x2, #0x70]\n"
+ "ldr x21, [x16, #0x48]\n"
+ "fmin z22.h, p3/M, z22.h, z14.h\n"
+ "ldr x20, [x16, #0x50]\n"
+ "fmin z21.h, p3/M, z21.h, z14.h\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "st1h { z20.h }, p0, [x19, x7, LSL #1]\n"
- "mov z20.d, z13.d\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "ldr x19, [x2, #0x78]\n"
+ "st1h { z22.h }, p0, [x21, x11, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
- "st1h { z19.h }, p0, [x22, x7, LSL #1]\n"
- "mov z19.d, z13.d\n"
+ "st1h { z21.h }, p0, [x20, x11, LSL #1]\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
+ "st1h { z20.h }, p0, [x19, x11, LSL #1]\n"
+ "fmax z16.h, p3/M, z16.h, z15.h\n"
+ "st1h { z19.h }, p0, [x22, x11, LSL #1]\n"
+ "ldr x21, [x16, #0x68]\n"
"fmin z18.h, p3/M, z18.h, z14.h\n"
- "st1h { z18.h }, p0, [x21, x7, LSL #1]\n"
- "mov z18.d, z13.d\n"
+ "ldr x20, [x16, #0x70]\n"
"fmin z17.h, p3/M, z17.h, z14.h\n"
- "st1h { z17.h }, p0, [x20, x7, LSL #1]\n"
- "mov z17.d, z13.d\n"
- "fmax z16.h, p3/M, z16.h, z15.h\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z16.h }, p0, [x19, x7, LSL #1]\n"
- "mov z16.d, z13.d\n"
+ "st1h { z18.h }, p0, [x21, x11, LSL #1]\n"
+ "st1h { z17.h }, p0, [x20, x11, LSL #1]\n"
+ "st1h { z16.h }, p0, [x19, x11, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0x20]\n"
- "inch x7\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0x28]\n"
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x20]\n"
+ "inch x11\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0x30]\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0x38]\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0x40]\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0x48]\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
- "ldr x28, [x4, #0x50]\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
- "ldr x27, [x4, #0x58]\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
- "ldr x26, [x4, #0x60]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
- "ldr x25, [x4, #0x68]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
- "ldr x24, [x4, #0x70]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ldr x23, [x4, #0x78]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "ldr x8, [x4, #0x80]\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
- "ldr x17, [x4, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
- "ldr x16, [x4, #0x90]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "ldr x15, [x4, #0x98]\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n"
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0xa0]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0xa8]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0xb0]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0xb8]\n"
"fmla z22.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0xc0]\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0xc8]\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "ldr x22, [x2, #0x0]\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x2, #0x8]\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
- "ldr x28, [x4, #0xd0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ldr x27, [x4, #0xd8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
- "ldr x26, [x4, #0xe0]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
- "ldr x20, [x2, #0x10]\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
- "ldr x19, [x2, #0x18]\n"
"fmla z22.h, p3/M, z5.h, z10.h\n"
"fmla z21.h, p3/M, z4.h, z10.h\n"
"fmla z20.h, p3/M, z3.h, z10.h\n"
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ldr x25, [x4, #0xe8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
- "ldr x23, [x4, #0xf8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
- "ldr x24, [x4, #0xf0]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
- "ldr x17, [x4, #0x108]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ldr x8, [x4, #0x100]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
- "ldr x15, [x4, #0x118]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
- "ldr x16, [x4, #0x110]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
@@ -679,68 +647,68 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(
"fmax z29.h, p3/M, z29.h, z15.h\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x22, x7, LSL #1]\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "ldr x22, [x2, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
- "st1h { z30.h }, p0, [x21, x7, LSL #1]\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
- "st1h { z29.h }, p0, [x20, x7, LSL #1]\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "st1h { z28.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
- "ldr x21, [x2, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
- "ldr x20, [x2, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
- "ldr x19, [x2, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z26.h, p3/M, z26.h, z14.h\n"
- "st1h { z27.h }, p0, [x22, x7, LSL #1]\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
"fmin z25.h, p3/M, z25.h, z14.h\n"
"fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z26.h }, p0, [x21, x7, LSL #1]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
- "ldr x22, [x2, #0x40]\n"
+ "ldr x22, [x16, #0x40]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
- "ldr x21, [x2, #0x48]\n"
+ "ldr x21, [x16, #0x48]\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
- "st1h { z25.h }, p0, [x20, x7, LSL #1]\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
- "st1h { z24.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
- "st1h { z23.h }, p0, [x22, x7, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
"fmin z22.h, p3/M, z22.h, z14.h\n"
- "ldr x20, [x2, #0x50]\n"
+ "ldr x20, [x16, #0x50]\n"
"fmin z21.h, p3/M, z21.h, z14.h\n"
- "ldr x19, [x2, #0x58]\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "ldr x22, [x2, #0x60]\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "st1h { z22.h }, p0, [x21, x7, LSL #1]\n"
+ "st1h { z22.h }, p0, [x21, x11, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
- "st1h { z21.h }, p0, [x20, x7, LSL #1]\n"
+ "st1h { z21.h }, p0, [x20, x11, LSL #1]\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
- "st1h { z20.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z20.h }, p0, [x19, x11, LSL #1]\n"
"fmax z16.h, p3/M, z16.h, z15.h\n"
- "st1h { z19.h }, p0, [x22, x7, LSL #1]\n"
- "ldr x21, [x2, #0x68]\n"
+ "st1h { z19.h }, p0, [x22, x11, LSL #1]\n"
+ "ldr x21, [x16, #0x68]\n"
"fmin z18.h, p3/M, z18.h, z14.h\n"
- "ldr x20, [x2, #0x70]\n"
+ "ldr x20, [x16, #0x70]\n"
"fmin z17.h, p3/M, z17.h, z14.h\n"
- "ldr x19, [x2, #0x78]\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z18.h }, p0, [x21, x7, LSL #1]\n"
- "st1h { z17.h }, p0, [x20, x7, LSL #1]\n"
- "st1h { z16.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z18.h }, p0, [x21, x11, LSL #1]\n"
+ "st1h { z17.h }, p0, [x20, x11, LSL #1]\n"
+ "st1h { z16.h }, p0, [x19, x11, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index a6a4afb3b5..5968309927 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst
+class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 2238bf08cd..d20e9913ae 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,225 +113,217 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(
"add x9, x21, x22, LSL #1\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x28, x9, x22, LSL #1\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
- "mov z31.d, z17.d\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z17.h }, p3/Z, [x16]\n"
"add x27, x28, x22, LSL #1\n"
- "mov z30.d, z17.d\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n"
"add x26, x27, x22, LSL #1\n"
- "mov z29.d, z17.d\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n"
"add x25, x13, x13\n"
- "mov z28.d, z17.d\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n"
"add x24, x25, x13\n"
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n"
"add x23, x24, x13\n"
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n"
"mul x19, x7, x20\n" // offset = tile_i * ld_output_row
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n"
"madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col
- "whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2)
- "ld1h { z10.h }, p2/Z, [x21]\n" // Load input point (0, 0)
+ "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n"
"mul x19, x19, x17\n" // offset *= output_tile_size
- "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1)
+ "whilelt p2.h, XZR, %x[n_channels]\n"
+ "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21]\n"
"add x10, x10, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
- "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"add x22, x10, x20, LSL #1\n"
- "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n"
"addvl x16, x16, #16\n"
- "ld1h { z14.h }, p2/Z, [x9]\n" // Load input point (1, 0)
+ "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n"
"cmp x14, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n"
"addvl x16, x16, #-6\n"
- "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z14.h }, p2/Z, [x9]\n"
+ "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
"whilelt p1.h, x14, %x[n_channels]\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
"inch x12\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z17.h }, p3/Z, [x16]\n"
"inch x15\n"
- "addvl x21, x21, #1\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x21]\n" // Load input point (0, 0)
+ "addvl x21, x21, #1\n"
+ "ld1h { z10.h }, p1/Z, [x21]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4)
"inch x14\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2)
- "addvl x9, x9, #1\n"
+ "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0)
+ "ld1h { z14.h }, p2/Z, [x27]\n"
+ "addvl x9, x9, #1\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0)
+ "ld1h { z15.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n"
"addvl x28, x28, #1\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0)
+ "ld1h { z15.h }, p2/Z, [x26]\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n"
"addvl x27, x27, #1\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n"
"fmin z31.h, p3/M, z31.h, z18.h\n"
- "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0)
- "mov z31.d, z17.d\n"
+ "st1h { z31.h }, p0, [x10]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4)
- "whilelt p2.h, x15, %x[n_channels]\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
- "addvl x26, x26, #1\n"
+ "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n"
+ "whilelt p2.h, x15, %x[n_channels]\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z17.d\n"
- "addvl x10, x10, #1\n"
+ "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n"
+ "addvl x26, x26, #1\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
- "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n"
"cmp x14, %x[n_channels]\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n"
+ "st1h { z30.h }, p0, [x10, x11, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
- "ld1h { z14.h }, p1/Z, [x9]\n" // Load input point (1, 0)
+ "ld1h { z14.h }, p1/Z, [x9]\n"
+ "addvl x10, x10, #1\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "mov z29.d, z17.d\n"
+ "st1h { z29.h }, p0, [x22]\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1)
- "addvl x16, x16, #16\n"
+ "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n"
+ "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
+ "addvl x16, x16, #16\n"
+ "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1)
- "mov z28.d, z17.d\n"
- "addvl x22, x22, #1\n"
+ "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n"
"addvl x16, x16, #-6\n"
+ "st1h { z28.h }, p0, [x22, x11, LSL #1]\n"
+ "addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
"ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
"ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x7, #0x1\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x8, x8, #0x1\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
"cmp x8, x19\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n"
"csel x8, x8, XZR, LT\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n"
"csel x7, x7, x21, LT\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0)
+ "ld1h { z14.h }, p2/Z, [x27]\n"
"cmp x7, x20\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0)
+ "ld1h { z15.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0)
+ "ld1h { z15.h }, p2/Z, [x26]\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
"fmin z31.h, p3/M, z31.h, z18.h\n"
- "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x10]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x10, x11, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
+ "st1h { z29.h }, p0, [x22]\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z28.h }, p0, [x22, x11, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
@@ -342,4 +334,4 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index 0d5d4176aa..ceba36d897 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -89,257 +89,249 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z19.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x14, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x13\n"
- "ldp x12, x11, [x19, #0x0]\n"
- "sub x10, XZR, x13\n"
- "ldp x9, x28, [x19, #0x10]\n"
+ "cnth x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
- "mov z31.d, z17.d\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
- "cmp x13, %x[n_channels]\n"
- "mov z30.d, z17.d\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
- "mov z29.d, z17.d\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
- "mov z28.d, z17.d\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #16\n"
- "ldp x27, x26, [x15, #0x0]\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #-6\n"
- "ld1h { z9.h }, p2/Z, [x27, x14, LSL #1]\n"
- "ld1h { z10.h }, p2/Z, [x26, x14, LSL #1]\n"
- "ldp x25, x23, [x15, #0x10]\n"
- "ldp x22, x21, [x15, #0x20]\n"
- "ldp x20, x19, [x15, #0x30]\n"
- "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x23, x14, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n"
- "ld1h { z14.h }, p2/Z, [x21, x14, LSL #1]\n"
- "ld1h { z15.h }, p2/Z, [x20, x14, LSL #1]\n"
- "ld1h { z16.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z17.h }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x19, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x24, [x15, #0x40]\n"
- "whilelt p1.h, x13, %x[n_channels]\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
- "ldr x20, [x15, #0x48]\n"
- "inch x10\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
- "ldr x23, [x15, #0x50]\n"
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x40]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
+ "ldr x25, [x14, #0x48]\n"
+ "inch x9\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
+ "ldr x24, [x14, #0x50]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ldr x19, [x15, #0x58]\n"
- "ldr x22, [x15, #0x60]\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ldr x21, [x15, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
- "ldr x27, [x15, #0x80]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ldr x26, [x15, #0x88]\n"
- "ldr x25, [x15, #0x90]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n"
- "ldr x23, [x15, #0x98]\n"
+ "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n"
- "ldr x22, [x15, #0xa0]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x21, [x15, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
- "ldr x20, [x15, #0xb0]\n"
- "ldr x19, [x15, #0xb8]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
- "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n"
- "ldr x24, [x15, #0xc0]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ldp x27, x26, [x15, #0x0]\n"
+ "ld1h { z17.h }, p3/Z, [x15]\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
- "ldp x25, x23, [x15, #0x10]\n"
- "ldp x22, x21, [x15, #0x20]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
- "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
- "ld1h { z9.h }, p1/Z, [x27, x13, LSL #1]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
- "ld1h { z10.h }, p1/Z, [x26, x13, LSL #1]\n"
- "ld1h { z12.h }, p1/Z, [x23, x13, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "fmin z31.h, p3/M, z31.h, z18.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
- "inch x14\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
- "ld1h { z13.h }, p1/Z, [x22, x13, LSL #1]\n"
- "whilelt p2.h, x14, %x[n_channels]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
+ "fmin z31.h, p3/M, z31.h, z18.h\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "ldp x20, x19, [x15, #0x30]\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
- "st1h { z31.h }, p0, [x12, x10, LSL #1]\n"
- "mov z31.d, z17.d\n"
- "ld1h { z16.h }, p1/Z, [x19, x13, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
- "st1h { z30.h }, p0, [x11, x10, LSL #1]\n"
- "mov z30.d, z17.d\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
- "ld1h { z14.h }, p1/Z, [x21, x13, LSL #1]\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n"
+ "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z14.h }, p1/Z, [x21, x12, LSL #1]\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p1/Z, [x20, x13, LSL #1]\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z15.h }, p1/Z, [x20, x12, LSL #1]\n"
+ "ld1h { z16.h }, p1/Z, [x19, x12, LSL #1]\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x25, x13, LSL #1]\n"
- "inch x13\n"
+ "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
- "st1h { z29.h }, p0, [x9, x10, LSL #1]\n"
- "cmp x13, %x[n_channels]\n"
- "mov z29.d, z17.d\n"
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #16\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #-6\n"
- "st1h { z28.h }, p0, [x28, x10, LSL #1]\n"
- "mov z28.d, z17.d\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x24, [x15, #0x40]\n"
- "inch x10\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
- "ldr x20, [x15, #0x48]\n"
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x40]\n"
+ "inch x9\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
+ "ldr x25, [x14, #0x48]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
- "ldr x23, [x15, #0x50]\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ldr x19, [x15, #0x58]\n"
- "ldr x22, [x15, #0x60]\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
+ "ldr x24, [x14, #0x50]\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ldr x21, [x15, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
- "ldr x27, [x15, #0x80]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ldr x26, [x15, #0x88]\n"
- "ldr x25, [x15, #0x90]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n"
- "ldr x23, [x15, #0x98]\n"
+ "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n"
- "ldr x22, [x15, #0xa0]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x21, [x15, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
- "ldr x20, [x15, #0xb0]\n"
- "ldr x19, [x15, #0xb8]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
- "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n"
- "ldr x24, [x15, #0xc0]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmin z31.h, p3/M, z31.h, z18.h\n"
- "st1h { z31.h }, p0, [x12, x10, LSL #1]\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "st1h { z30.h }, p0, [x11, x10, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "st1h { z29.h }, p0, [x9, x10, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z28.h }, p0, [x28, x10, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 236f9bf43a..4a9bd33a1e 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst
+class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index 6b1564e6c9..1c2e1e27ad 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,390 +113,382 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
"add x20, x13, x22, LSL #1\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x10, x20, x22, LSL #1\n"
- "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x8]\n"
"add x9, x10, x22, LSL #1\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n"
"add x28, x9, x22, LSL #1\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n"
"add x27, x28, x22, LSL #1\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n"
"add x26, x15, x15\n"
- "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n"
"add x25, x26, x15\n"
- "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
+ "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n"
"add x24, x25, x15\n"
+ "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
"add x23, x24, x15\n"
"madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col
"mul x19, x19, x7\n" // offset *= output_tile_size
"add x11, x11, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
"add x22, x11, x21, LSL #1\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z5.h }, p2/Z, [x13]\n" // Load input point (0, 0)
- "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z5.h }, p2/Z, [x13]\n"
+ "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n"
"cmp x16, %x[n_channels]\n"
- "ld1h { z7.h }, p2/Z, [x20]\n" // Load input point (1, 0)
+ "ld1h { z7.h }, p2/Z, [x20]\n"
"addvl x8, x8, #6\n"
- "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2)
- "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2)
- "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3)
- "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4)
- "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5)
- "ld1h { z14.h }, p2/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x10]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3)
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n"
"whilelt p1.h, x16, %x[n_channels]\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
"inch x14\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z0.h }, p3/Z, [x8]\n"
"inch x17\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n"
"addvl x20, x20, #1\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
"inch x16\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n"
"addvl x13, x13, #1\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
- "ld1h { z7.h }, p1/Z, [x20]\n" // Load input point (1, 0)
+ "ld1h { z7.h }, p1/Z, [x20]\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n"
"addvl x10, x10, #1\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x8, x8, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0)
- "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p2/Z, [x9]\n"
+ "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n"
"addvl x9, x9, #1\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0)
+ "ld1h { z9.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n"
"addvl x28, x28, #1\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
- "ld1h { z14.h }, p1/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z14.h }, p1/Z, [x10]\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
- "ld1h { z5.h }, p1/Z, [x13]\n" // Load input point (0, 0)
+ "ld1h { z5.h }, p1/Z, [x13]\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n"
"whilelt p2.h, x17, %x[n_channels]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x27, x27, #1\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
- "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n"
"addvl x8, x8, #16\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
"cmp x16, %x[n_channels]\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5)
+ "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n"
"fmla z28.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2)
- "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n"
+ "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
"addvl x8, x8, #-6\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0)
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x11]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z16.d\n"
- "addvl x11, x11, #1\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "mov z29.d, z16.d\n"
+ "st1h { z30.h }, p0, [x11, x12, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1)
- "mov z28.d, z16.d\n"
+ "addvl x11, x11, #1\n"
+ "st1h { z29.h }, p0, [x22]\n"
+ "st1h { z28.h }, p0, [x22, x12, LSL #1]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3)
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
"ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"add x21, x5, #0x1\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
"ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z0.h }, p3/Z, [x8]\n"
"add x6, x6, #0x1\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"cmp x6, x19\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n"
"csel x6, x6, XZR, LT\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"csel x5, x5, x21, LT\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
"cmp x5, x20\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x8, x8, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0)
+ "ld1h { z5.h }, p2/Z, [x9]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0)
+ "ld1h { z9.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
@@ -511,13 +503,13 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x11]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x11, x12, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z29.h }, p0, [x22]\n"
+ "st1h { z28.h }, p0, [x22, x12, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
@@ -528,4 +520,4 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index be128b4aff..a0640daeca 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -100,432 +100,424 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x5, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x6, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x7, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x8\n"
- "ldp x17, x16, [x19, #0x0]\n"
- "sub x15, XZR, x8\n"
- "ldp x14, x13, [x19, #0x10]\n"
+ "cnth x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z16.h }, p3/Z, [x5]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
- "cmp x8, %x[n_channels]\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
- "ld1h { z4.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
- "addvl x5, x5, #6\n"
- "ldp x12, x11, [x6, #0x0]\n"
- "ldp x10, x9, [x6, #0x10]\n"
- "ldp x20, x28, [x6, #0x20]\n"
- "ld1h { z5.h }, p2/Z, [x12, x7, LSL #1]\n"
- "ld1h { z6.h }, p2/Z, [x11, x7, LSL #1]\n"
- "ld1h { z7.h }, p2/Z, [x10, x7, LSL #1]\n"
- "ld1h { z8.h }, p2/Z, [x9, x7, LSL #1]\n"
- "ld1h { z9.h }, p2/Z, [x20, x7, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x28, x7, LSL #1]\n"
- "ldp x27, x19, [x6, #0x30]\n"
- "ldp x26, x25, [x6, #0x40]\n"
- "ld1h { z11.h }, p2/Z, [x27, x7, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n"
- "ld1h { z10.h }, p2/Z, [x26, x7, LSL #1]\n"
- "ld1h { z14.h }, p2/Z, [x25, x7, LSL #1]\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #6\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ld1h { z5.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ld1h { z7.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ldr x24, [x6, #0x50]\n"
- "whilelt p1.h, x8, %x[n_channels]\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
- "ldr x23, [x6, #0x58]\n"
- "inch x15\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
- "ldr x22, [x6, #0x60]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ldr x24, [x14, #0x50]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
+ "ldr x23, [x14, #0x58]\n"
+ "inch x9\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
+ "ldr x22, [x14, #0x60]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
- "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z0.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x6, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
- "ldr x20, [x6, #0x70]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ldr x19, [x6, #0x78]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
- "ldr x12, [x6, #0x80]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
- "ldr x11, [x6, #0x88]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
- "ldr x10, [x6, #0x90]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
- "ldr x9, [x6, #0x98]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
- "ldr x20, [x6, #0xa0]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
- "ldr x28, [x6, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
- "ldr x27, [x6, #0xb0]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
- "ldr x19, [x6, #0xb8]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
- "ldr x26, [x6, #0xc0]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
- "ldr x25, [x6, #0xc8]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
- "ldr x24, [x6, #0xd0]\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "addvl x5, x5, #16\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
- "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n"
- "ldr x23, [x6, #0xd8]\n"
+ "ld1h { z16.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
- "ldr x22, [x6, #0xe0]\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
- "ldr x21, [x6, #0xe8]\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
- "ldr x20, [x6, #0xf0]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
- "ldr x19, [x6, #0xf8]\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ldr x12, [x6, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
- "ldr x11, [x6, #0x108]\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
- "ldr x10, [x6, #0x110]\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
- "ldr x9, [x6, #0x118]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x100]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z16.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x25, [x14, #0x108]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0x110]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x118]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
- "ldp x12, x11, [x6, #0x0]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
- "ld1h { z5.h }, p1/Z, [x12, x8, LSL #1]\n"
+ "ld1h { z5.h }, p1/Z, [x26, x12, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n"
- "inch x7\n"
+ "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ldp x10, x9, [x6, #0x10]\n"
- "whilelt p2.h, x7, %x[n_channels]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
- "ld1h { z6.h }, p1/Z, [x11, x8, LSL #1]\n"
- "ldp x20, x28, [x6, #0x20]\n"
+ "ld1h { z6.h }, p1/Z, [x25, x12, LSL #1]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
- "ldp x27, x19, [x6, #0x30]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
- "ld1h { z7.h }, p1/Z, [x10, x8, LSL #1]\n"
+ "ld1h { z7.h }, p1/Z, [x24, x12, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z13.h }, p1/Z, [x28, x8, LSL #1]\n"
+ "ld1h { z13.h }, p1/Z, [x21, x12, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p1/Z, [x9, x8, LSL #1]\n"
+ "ld1h { z8.h }, p1/Z, [x23, x12, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z11.h }, p1/Z, [x27, x8, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x20, x12, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x19, x8, LSL #1]\n"
+ "ld1h { z12.h }, p1/Z, [x19, x12, LSL #1]\n"
"fmla z28.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x20, x8, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x22, x12, LSL #1]\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
- "ldp x26, x25, [x6, #0x40]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
- "addvl x5, x5, #16\n"
+ "addvl x15, x15, #16\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "ld1h { z10.h }, p1/Z, [x26, x8, LSL #1]\n"
- "ld1h { z14.h }, p1/Z, [x25, x8, LSL #1]\n"
+ "ld1h { z10.h }, p1/Z, [x26, x12, LSL #1]\n"
+ "ld1h { z14.h }, p1/Z, [x25, x12, LSL #1]\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "inch x8\n"
+ "inch x12\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x8, %x[n_channels]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x5, x5, #-6\n"
+ "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z31.h }, p0, [x17, x15, LSL #1]\n"
- "mov z31.d, z16.d\n"
- "st1h { z30.h }, p0, [x16, x15, LSL #1]\n"
- "mov z30.d, z16.d\n"
- "st1h { z29.h }, p0, [x14, x15, LSL #1]\n"
- "mov z29.d, z16.d\n"
- "st1h { z28.h }, p0, [x13, x15, LSL #1]\n"
- "mov z28.d, z16.d\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ldr x24, [x6, #0x50]\n"
- "inch x15\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
- "ldr x23, [x6, #0x58]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ldr x24, [x14, #0x50]\n"
+ "inch x9\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
+ "ldr x23, [x14, #0x58]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
- "ldr x22, [x6, #0x60]\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
- "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
+ "ldr x22, [x14, #0x60]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z0.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x6, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
- "ldr x20, [x6, #0x70]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ldr x19, [x6, #0x78]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
- "ldr x12, [x6, #0x80]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
- "ldr x11, [x6, #0x88]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
- "ldr x10, [x6, #0x90]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
- "ldr x9, [x6, #0x98]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
- "ldr x20, [x6, #0xa0]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
- "ldr x28, [x6, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
- "ldr x27, [x6, #0xb0]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
- "ldr x19, [x6, #0xb8]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias
- "ldr x26, [x6, #0xc0]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
- "ldr x25, [x6, #0xc8]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias
- "addvl x5, x5, #16\n"
+ "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n"
- "ldr x24, [x6, #0xd0]\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
- "ldr x23, [x6, #0xd8]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias
- "ldr x22, [x6, #0xe0]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias
- "ldr x21, [x6, #0xe8]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
- "ldr x20, [x6, #0xf0]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias
- "ldr x19, [x6, #0xf8]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ldr x12, [x6, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias
- "ldr x11, [x6, #0x108]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
- "ldr x10, [x6, #0x110]\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias
- "ldr x9, [x6, #0x118]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x100]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x25, [x14, #0x108]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0x110]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x118]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
@@ -540,20 +532,20 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x17, x15, LSL #1]\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x16, x15, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z29.h }, p0, [x14, x15, LSL #1]\n"
- "st1h { z28.h }, p0, [x13, x15, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index 05e82d4e76..e07e6314f8 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst
+class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index eb632eb4fe..571246be3e 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -88,277 +88,229 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(
__asm__ __volatile__(
"ptrue p3.b\n"
- "mov x8, #0x0\n"
"mov x17, #0x0\n"
+ "mov x16, #0x0\n"
"1:" // Tile loop
- "str x8, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "mov x21, #0x2\n"
- "str x17, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "cntb x16\n"
- "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x16, x16, XZR, LSL #4\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
- "cntb x14\n"
- "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
- "cntb x12\n"
- "ldr x11, [%x[params_struct], %[offsetof_args_inptr]]\n"
- "mul x19, x8, x20\n" // offset = tile_i * ld_input_row
- "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
- "madd x19, x17, x13, x19\n" // offset += tile_j * ld_input_col
- "ldr x10, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
- "mul x19, x19, x21\n" // offset *= kernel_stride * output_size
- "ldr x9, [%x[params_struct], %[offsetof_args_outptr]]\n"
- "add x11, x11, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
+ "str x17, [%x[params_struct], %[offsetof_args_tile_i]]\n"
+ "mov x23, #0x2\n"
+ "str x16, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "mov x15, #0x2\n"
+ "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n"
+ "mov x13, #0x0\n"
+ "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
+ "cntw x12\n"
+ "ldr x11, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
+ "sub x21, XZR, x12\n"
+ "ldr x10, [%x[params_struct], %[offsetof_args_inptr]]\n"
+ "mul x19, x17, x22\n" // offset = tile_i * ld_input_row
+ "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
+ "madd x19, x16, x11, x19\n" // offset += tile_j * ld_input_col
+ "ldr x9, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
+ "mul x19, x19, x23\n" // offset *= kernel_stride * output_size
+ "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n"
+ "add x10, x10, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "add x28, x11, x20, LSL #2\n"
+ "add x27, x10, x22, LSL #2\n"
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "add x27, x28, x20, LSL #2\n"
- "ld1w { z16.s }, p3/Z, [x15]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
- "add x26, x27, x20, LSL #2\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
- "add x25, x13, x13\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
- "add x24, x25, x13\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
- "add x14, x14, x13, LSL #4\n"
- "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
- "add x12, x12, x25, LSL #4\n"
- "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
- "cntb x23\n"
- "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
- "add x23, x23, x24, LSL #4\n"
- "prfm pldl1keep, [x28, x14]\n"
- "mov x20, #0x2\n"
- "prfm pldl1keep, [x11, x16]\n"
- "mul x19, x8, x22\n" // offset = tile_i * ld_output_row
- "prfm pldl1keep, [x11, x23]\n"
- "madd x19, x17, x10, x19\n" // offset += tile_j * ld_output_col
- "prfm pldl1keep, [x28, x12]\n"
- "mul x19, x19, x20\n" // offset *= output_tile_size
- "prfm pldl1keep, [x27, x14]\n"
- "add x9, x9, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "mov x21, #0x0\n"
- "add x22, x9, x22, LSL #2\n"
- "cntw x20\n"
- "sub x19, XZR, x20\n"
+ "add x26, x27, x22, LSL #2\n"
+ "ld1w { z16.s }, p3/Z, [x14]\n"
+ "add x25, x26, x22, LSL #2\n"
+ "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
+ "add x24, x11, x11\n"
+ "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
+ "add x23, x24, x11\n"
+ "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
+ "mul x19, x17, x20\n" // offset = tile_i * ld_output_row
+ "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
+ "madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col
+ "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
+ "mul x19, x19, x15\n" // offset *= output_tile_size
+ "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
+ "add x28, x28, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
+ "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
+ "add x22, x28, x20, LSL #2\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x11]\n"
- "addvl x15, x15, #16\n"
- "ld1w { z11.s }, p2/Z, [x11, x24, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
- "addvl x15, x15, #-6\n"
- "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n"
- "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x27, x11, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10]\n"
+ "addvl x14, x14, #16\n"
+ "ld1w { z11.s }, p2/Z, [x10, x23, LSL #2]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
+ "addvl x14, x14, #-6\n"
+ "ld1w { z12.s }, p2/Z, [x27, x24, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x26, x11, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x16]\n"
- "whilelt p1.s, x20, %x[n_channels]\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x23]\n"
- "incw x19\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x14]\n"
- "mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
"incw x21\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "mov p0.b, p2.b\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x25]\n"
+ "incw x13\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x11, x12]\n"
- "incw x20\n"
+ "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n"
+ "incw x12\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x27, x12]\n"
+ "ld1w { z16.s }, p3/Z, [x14]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x28, x16]\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x28, x23]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n"
+ "addvl x10, x10, #1\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "addvl x11, x11, #1\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x16]\n"
- "prfm pldl1keep, [x27, x23]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
- "prfm pldl1keep, [x26, x14]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x26, x12]\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x28]\n"
+ "ld1w { z11.s }, p2/Z, [x27]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "ld1w { z16.s }, p3/Z, [x15]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n"
+ "addvl x27, x27, #1\n"
"fmla z29.s, p3/M, z5.s, z10.s\n"
- "addvl x28, x28, #1\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x28, x14]\n"
- "prfm pldl1keep, [x11, x16]\n"
+ "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x11, x23]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x27]\n"
+ "ld1w { z9.s }, p2/Z, [x26]\n"
+ "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "prfm pldl1keep, [x28, x12]\n"
+ "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
- "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
"fmla z31.s, p3/M, z8.s, z10.s\n"
- "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n"
+ "addvl x26, x26, #1\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
- "addvl x27, x27, #1\n"
+ "ld1w { z13.s }, p1/Z, [x26, x11, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
- "ld1w { z13.s }, p1/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n"
- "whilelt p2.s, x21, %x[n_channels]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x27, x14]\n"
- "addvl x26, x26, #1\n"
+ "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
+ "addvl x25, x25, #1\n"
"fmla z31.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p1/Z, [x28, x13, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
+ "ld1w { z9.s }, p1/Z, [x27, x11, LSL #2]\n"
+ "cmp x12, %x[n_channels]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p1/Z, [x11]\n"
+ "ld1w { z10.s }, p1/Z, [x10]\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x11, x24, LSL #2]\n"
- "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x10, x23, LSL #2]\n"
+ "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "addvl x14, x14, #16\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
+ "ld1w { z12.s }, p1/Z, [x27, x24, LSL #2]\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x28, x25, LSL #2]\n"
- "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
+ "addvl x14, x14, #-6\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "addvl x15, x15, #16\n"
+ "st1w { z31.s }, p0, [x28]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
- "addvl x15, x15, #-6\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z31.s }, p0, [x9]\n"
- "mov z31.d, z16.d\n"
+ "st1w { z30.s }, p0, [x28, x9, LSL #2]\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
+ "addvl x28, x28, #1\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z30.s }, p0, [x9, x10, LSL #2]\n"
- "addvl x9, x9, #1\n"
- "mov z30.d, z16.d\n"
"st1w { z29.s }, p0, [x22]\n"
- "mov z29.d, z16.d\n"
- "st1w { z28.s }, p0, [x22, x10, LSL #2]\n"
- "mov z28.d, z16.d\n"
+ "st1w { z28.s }, p0, [x22, x9, LSL #2]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x16]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x23]\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x14]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26]\n"
- "prfm pldl1keep, [x11, x12]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "add x21, x17, #0x1\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x25]\n"
+ "add x16, x16, #0x1\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n"
- "fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x27, x12]\n"
- "prfm pldl1keep, [x28, x16]\n"
+ "cmp x16, x19\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x28, x23]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
+ "csel x16, x16, XZR, LT\n"
+ "fmla z28.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n"
+ "csel x17, x17, x21, LT\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z3.s, z13.s\n"
- "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n"
- "prfm pldl1keep, [x27, x16]\n"
+ "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n"
+ "cmp x17, x20\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x23]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
+ "fmla z28.s, p3/M, z3.s, z13.s\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x28]\n"
- "prfm pldl1keep, [x26, x12]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "ldr x8, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "add x21, x8, #0x1\n"
+ "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x27]\n"
"fmla z29.s, p3/M, z5.s, z10.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "ldr x17, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "add x17, x17, #0x1\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x27]\n"
- "fmla z31.s, p3/M, z8.s, z10.s\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
+ "ld1w { z9.s }, p2/Z, [x26]\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
- "cmp x17, x19\n"
- "fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n"
- "fmla z31.s, p3/M, z3.s, z11.s\n"
- "csel x17, x17, XZR, LT\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
- "csel x8, x8, x21, LT\n"
+ "fmla z31.s, p3/M, z8.s, z10.s\n"
+ "fmla z30.s, p3/M, z7.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
- "cmp x8, x20\n"
- "fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n"
- "fmla z31.s, p3/M, z6.s, z9.s\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
+ "fmla z31.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n"
+ "fmla z30.s, p3/M, z5.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
- "fmla z30.s, p3/M, z8.s, z10.s\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
- "fmax z31.s, p3/M, z31.s, z18.s\n"
+ "fmla z31.s, p3/M, z6.s, z9.s\n"
+ "fmla z30.s, p3/M, z8.s, z10.s\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
- "fmax z30.s, p3/M, z30.s, z18.s\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x9]\n"
+ "fmax z31.s, p3/M, z31.s, z18.s\n"
+ "fmax z30.s, p3/M, z30.s, z18.s\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z30.s }, p0, [x9, x10, LSL #2]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
+ "fmin z31.s, p3/M, z31.s, z17.s\n"
+ "st1w { z31.s }, p0, [x28]\n"
+ "fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
+ "st1w { z30.s }, p0, [x28, x9, LSL #2]\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
"st1w { z29.s }, p0, [x22]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z28.s }, p0, [x22, x10, LSL #2]\n"
+ "st1w { z28.s }, p0, [x22, x9, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index fb41ca0754..77a6c683b0 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -54,22 +54,22 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
const float max
) : outptrs(outptrs), params(params), min(min), max(max)
{
- inptrs[0] = input_ptrs[0];
- inptrs[1] = input_ptrs[1];
- inptrs[2] = input_ptrs[2];
- inptrs[3] = input_ptrs[3];
- inptrs[4] = input_ptrs[4];
- inptrs[5] = input_ptrs[5];
- inptrs[6] = input_ptrs[6];
- inptrs[7] = input_ptrs[7];
- inptrs[8] = input_ptrs[8];
- inptrs[9] = input_ptrs[9];
- inptrs[10] = input_ptrs[10];
- inptrs[11] = input_ptrs[11];
- inptrs[12] = input_ptrs[12];
- inptrs[13] = input_ptrs[13];
- inptrs[14] = input_ptrs[14];
- inptrs[15] = input_ptrs[15];
+ inptrs[0] = input_ptrs[5];
+ inptrs[1] = input_ptrs[0];
+ inptrs[2] = input_ptrs[3];
+ inptrs[3] = input_ptrs[6];
+ inptrs[4] = input_ptrs[9];
+ inptrs[5] = input_ptrs[12];
+ inptrs[6] = input_ptrs[15];
+ inptrs[7] = input_ptrs[1];
+ inptrs[8] = input_ptrs[2];
+ inptrs[9] = input_ptrs[10];
+ inptrs[10] = input_ptrs[4];
+ inptrs[11] = input_ptrs[7];
+ inptrs[12] = input_ptrs[8];
+ inptrs[13] = input_ptrs[11];
+ inptrs[14] = input_ptrs[13];
+ inptrs[15] = input_ptrs[14];
}
};
@@ -78,216 +78,194 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x4, ALL, MUL #2\n"
- "ldp x5, x6, [x19, #0x0]\n"
- "mov x7, #0x0\n"
- "ldp x8, x17, [x19, #0x10]\n"
- "cntw x16\n"
- "ldp x15, x14, [x19, #0x20]\n"
- "sub x13, XZR, x16\n"
- "ldp x12, x11, [x19, #0x30]\n"
- "whilelt p2.s, XZR, %x[n_channels]\n"
- "ldp x10, x9, [x19, #0x40]\n"
- "cmp x16, %x[n_channels]\n"
- "ldp x28, x27, [x19, #0x50]\n"
- "ldp x26, x25, [x19, #0x60]\n"
- "ldp x24, x23, [x19, #0x70]\n"
- "ldp x22, x21, [x2, #0x0]\n"
- "ldp x20, x19, [x2, #0x10]\n"
+ "mov x13, #0x0\n"
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "ld1w { z16.s }, p3/Z, [x3]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n"
- "addvl x3, x3, #16\n"
- "ld1w { z9.s }, p2/Z, [x14, x7, LSL #2]\n"
- "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n"
- "addvl x3, x3, #-6\n"
- "prfm pldl1keep, [x14, x4]\n"
- "ld1w { z10.s }, p2/Z, [x5, x7, LSL #2]\n"
- "prfm pldl1keep, [x5, x4]\n"
- "ld1w { z11.s }, p2/Z, [x17, x7, LSL #2]\n"
- "prfm pldl1keep, [x17, x4]\n"
- "ld1w { z12.s }, p2/Z, [x12, x7, LSL #2]\n"
- "prfm pldl1keep, [x12, x4]\n"
- "ld1w { z13.s }, p2/Z, [x9, x7, LSL #2]\n"
- "prfm pldl1keep, [x9, x4]\n"
+ "cntw x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
+ "whilelt p2.s, XZR, %x[n_channels]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldr x22, [x14, #0x20]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x4]\n"
- "whilelt p1.s, x16, %x[n_channels]\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x23, x4]\n"
- "incw x13\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x6, x4]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "ldr x21, [x14, #0x28]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
+ "ldr x20, [x14, #0x30]\n"
+ "incw x9\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "ldr x19, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n"
- "prfm pldl1keep, [x8, x4]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x28, x4]\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x15, x4]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x11, x4]\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x10, x4]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x4]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
- "prfm pldl1keep, [x25, x4]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x24, x4]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "addvl x4, x4, #1\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n"
- "prfm pldl1keep, [x14, x4]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x5, x4]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x17, x4]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x12, x4]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z13.s }, p1/Z, [x9, x16, LSL #2]\n"
+ "ldr x22, [x14, #0x20]\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x9, x4]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.s, p3/M, z8.s, z10.s\n"
- "ld1w { z16.s }, p3/Z, [x3]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
- "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n"
+ "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
- "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n"
- "incw x7\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
- "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n"
- "whilelt p2.s, x7, %x[n_channels]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
"fmla z31.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x17, x16, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p1/Z, [x5, x16, LSL #2]\n"
- "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n"
+ "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
+ "addvl x15, x15, #16\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x12, x16, LSL #2]\n"
- "incw x16\n"
+ "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n"
+ "incw x12\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n"
- "cmp x16, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n"
- "addvl x3, x3, #16\n"
- "fmin z31.s, p3/M, z31.s, z17.s\n"
- "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n"
- "addvl x3, x3, #-6\n"
+ "fmin z31.s, p3/M, z31.s, z17.s\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z31.s }, p0, [x22, x13, LSL #2]\n"
- "mov z31.d, z16.d\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x21, x13, LSL #2]\n"
- "mov z30.d, z16.d\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z29.s }, p0, [x20, x13, LSL #2]\n"
- "mov z29.d, z16.d\n"
- "st1w { z28.s }, p0, [x19, x13, LSL #2]\n"
- "mov z28.d, z16.d\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x4]\n"
- "incw x13\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x23, x4]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "ldr x21, [x14, #0x28]\n"
+ "incw x9\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
+ "ldr x20, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x6, x4]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n"
- "prfm pldl1keep, [x8, x4]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "ldr x19, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x28, x4]\n"
- "prfm pldl1keep, [x15, x4]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x11, x4]\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n"
- "prfm pldl1keep, [x10, x4]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x4]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
- "prfm pldl1keep, [x25, x4]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n"
- "prfm pldl1keep, [x24, x4]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z10.s\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
"fmla z31.s, p3/M, z8.s, z10.s\n"
"fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
"fmla z31.s, p3/M, z6.s, z9.s\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
@@ -298,21 +276,21 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(
"fmax z30.s, p3/M, z30.s, z18.s\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x22, x13, LSL #2]\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x21, x13, LSL #2]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z29.s }, p0, [x20, x13, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z28.s }, p0, [x19, x13, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index ef5f4187f9..eb9de9fa01 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst
+class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index 6bc333be41..c485b7dde0 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -88,167 +88,123 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
__asm__ __volatile__(
"ptrue p3.b\n"
- "mov x3, #0x0\n"
- "mov x4, #0x0\n"
+ "mov x6, #0x0\n"
+ "mov x7, #0x0\n"
"1:" // Tile loop
- "str x3, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "mov x22, #0x3\n"
- "str x4, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "cntb x5\n"
- "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x5, x5, XZR, LSL #4\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
- "cntb x7\n"
- "ldr x8, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
- "cntb x17\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_inptr]]\n"
- "mul x19, x3, x20\n" // offset = tile_i * ld_input_row
- "ldr x21, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
- "madd x19, x4, x8, x19\n" // offset += tile_j * ld_input_col
- "ldr x15, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
- "mul x19, x19, x22\n" // offset *= kernel_stride * output_size
- "ldr x14, [%x[params_struct], %[offsetof_args_outptr]]\n"
- "add x16, x16, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
+ "str x6, [%x[params_struct], %[offsetof_args_tile_i]]\n"
+ "mov x24, #0x3\n"
+ "str x7, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "mov x23, #0x3\n"
+ "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n"
+ "mov x17, #0x0\n"
+ "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
+ "cntw x16\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
+ "sub x21, XZR, x16\n"
+ "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n"
+ "mul x19, x6, x22\n" // offset = tile_i * ld_input_row
+ "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
+ "madd x19, x7, x15, x19\n" // offset += tile_j * ld_input_col
+ "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
+ "mul x19, x19, x24\n" // offset *= kernel_stride * output_size
+ "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n"
+ "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "add x13, x16, x20, LSL #2\n"
+ "add x11, x14, x22, LSL #2\n"
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "add x12, x13, x20, LSL #2\n"
- "ld1w { z16.s }, p3/Z, [x6]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
- "add x11, x12, x20, LSL #2\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
- "add x10, x11, x20, LSL #2\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
- "add x9, x8, x8\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
- "add x28, x9, x8\n"
- "mov z27.d, z16.d\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
- "add x27, x28, x8\n"
- "mov z26.d, z16.d\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
- "add x7, x7, x8, LSL #4\n"
- "mov z25.d, z16.d\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
- "add x17, x17, x9, LSL #4\n"
- "mov z24.d, z16.d\n"
- "prfm pldl1keep, [x12, x17]\n"
- "cntb x26\n"
- "mov z23.d, z16.d\n"
- "prfm pldl1keep, [x16, x5]\n"
- "add x26, x26, x28, LSL #4\n"
- "cntb x25\n"
- "mov x20, #0x3\n"
- "add x25, x25, x27, LSL #4\n"
- "prfm pldl1keep, [x16, x25]\n"
- "prfm pldl1keep, [x10, x5]\n"
- "mul x19, x3, x21\n" // offset = tile_i * ld_output_row
- "prfm pldl1keep, [x13, x17]\n"
- "madd x19, x4, x15, x19\n" // offset += tile_j * ld_output_col
- "add x24, x15, x15\n"
- "mul x19, x19, x20\n" // offset *= output_tile_size
- "add x14, x14, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "add x23, x14, x21, LSL #2\n"
- "add x22, x23, x21, LSL #2\n"
- "mov x21, #0x0\n"
- "cntw x20\n"
- "sub x19, XZR, x20\n"
+ "add x10, x11, x22, LSL #2\n"
+ "ld1w { z16.s }, p3/Z, [x8]\n"
+ "add x9, x10, x22, LSL #2\n"
+ "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
+ "add x28, x9, x22, LSL #2\n"
+ "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
+ "add x27, x15, x15\n"
+ "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
+ "add x26, x27, x15\n"
+ "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
+ "add x25, x26, x15\n"
+ "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
+ "mul x19, x6, x20\n" // offset = tile_i * ld_output_row
+ "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
+ "madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col
+ "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
+ "mul x19, x19, x23\n" // offset *= output_tile_size
+ "add x24, x13, x13\n"
+ "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
+ "add x23, x12, x20, LSL #2\n"
+ "add x22, x23, x20, LSL #2\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "ld1w { z9.s }, p2/Z, [x12, x9, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x16]\n"
- "addvl x6, x6, #16\n"
- "ld1w { z11.s }, p2/Z, [x16, x27, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
- "ld1w { z12.s }, p2/Z, [x10]\n"
- "ld1w { z13.s }, p2/Z, [x13, x9, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x14]\n"
+ "addvl x8, x8, #16\n"
+ "ld1w { z11.s }, p2/Z, [x14, x25, LSL #2]\n"
+ "cmp x16, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
+ "addvl x8, x8, #-6\n"
+ "ld1w { z12.s }, p2/Z, [x28]\n"
+ "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x10, x25]\n"
- "whilelt p1.s, x20, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "prfm pldl1keep, [x12, x7]\n"
- "incw x19\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x16, x7]\n"
- "mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x16, x26]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "whilelt p1.s, x16, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
"incw x21\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x12, x26]\n"
- "incw x20\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x13, x5]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x13, x25]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x5]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x11, x17]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "mov p0.b, p2.b\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "incw x17\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "incw x16\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z16.s }, p3/Z, [x8]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n"
"fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x11, x25]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x10, x7]\n"
"fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x13, x7]\n"
"fmla z28.s, p3/M, z2.s, z13.s\n"
- "prfm pldl1keep, [x13, x26]\n"
"fmla z27.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x26]\n"
"fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n"
"fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x7]\n"
"fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x16, x17]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x11, x26]\n"
"fmla z27.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x12, x5]\n"
"fmla z25.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x12, x25]\n"
"fmla z24.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13]\n"
+ "ld1w { z11.s }, p2/Z, [x11]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x17]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "ld1w { z16.s }, p3/Z, [x6]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11]\n"
+ "ld1w { z12.s }, p2/Z, [x9]\n"
"fmla z29.s, p3/M, z7.s, z10.s\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
@@ -258,198 +214,164 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"fmla z26.s, p3/M, z8.s, z11.s\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n"
- "addvl x13, x13, #1\n"
+ "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
+ "addvl x11, x11, #1\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n"
- "addvl x16, x16, #1\n"
+ "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n"
+ "addvl x14, x14, #1\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
- "ld1w { z10.s }, p1/Z, [x16]\n"
+ "ld1w { z10.s }, p1/Z, [x14]\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n"
- "addvl x11, x11, #1\n"
+ "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
+ "addvl x9, x9, #1\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12]\n"
+ "ld1w { z12.s }, p2/Z, [x10]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n"
- "addvl x12, x12, #1\n"
+ "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
+ "addvl x10, x10, #1\n"
"fmla z27.s, p3/M, z8.s, z13.s\n"
- "ld1w { z9.s }, p1/Z, [x12, x9, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n"
"fmla z26.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x12, x17]\n"
"fmla z24.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x16, x5]\n"
"fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n"
- "whilelt p2.s, x21, %x[n_channels]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n"
+ "whilelt p2.s, x17, %x[n_channels]\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
- "prfm pldl1keep, [x16, x25]\n"
- "addvl x10, x10, #1\n"
+ "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
+ "addvl x28, x28, #1\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x10, x5]\n"
- "cmp x20, %x[n_channels]\n"
+ "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
+ "cmp x16, %x[n_channels]\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p1/Z, [x10]\n"
+ "ld1w { z12.s }, p1/Z, [x28]\n"
"fmla z29.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x13, x17]\n"
+ "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z26.s, p3/M, z5.s, z11.s\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z23.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x16, x27, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x14, x25, LSL #2]\n"
"fmla z24.s, p3/M, z7.s, z13.s\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z25.s, p3/M, z8.s, z13.s\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
"fmla z23.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p1/Z, [x13, x9, LSL #2]\n"
+ "ld1w { z13.s }, p1/Z, [x11, x27, LSL #2]\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
- "fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
- "addvl x6, x6, #16\n"
+ "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
+ "addvl x8, x8, #16\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
+ "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
+ "fmax z29.s, p3/M, z29.s, z18.s\n"
+ "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
+ "addvl x8, x8, #-6\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z31.s }, p0, [x14]\n"
- "mov z31.d, z16.d\n"
+ "st1w { z31.s }, p0, [x12]\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x14, x15, LSL #2]\n"
- "mov z30.d, z16.d\n"
- "fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z29.s }, p0, [x14, x24, LSL #2]\n"
- "mov z29.d, z16.d\n"
- "addvl x14, x14, #1\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
+ "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
- "st1w { z28.s }, p0, [x23]\n"
- "mov z28.d, z16.d\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
+ "st1w { z29.s }, p0, [x12, x24, LSL #2]\n"
+ "fmin z28.s, p3/M, z28.s, z17.s\n"
+ "addvl x12, x12, #1\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
- "fmax z24.s, p3/M, z24.s, z18.s\n"
+ "st1w { z28.s }, p0, [x23]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
- "st1w { z27.s }, p0, [x23, x15, LSL #2]\n"
- "mov z27.d, z16.d\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
+ "st1w { z27.s }, p0, [x23, x13, LSL #2]\n"
+ "fmin z25.s, p3/M, z25.s, z17.s\n"
+ "fmax z24.s, p3/M, z24.s, z18.s\n"
"st1w { z26.s }, p0, [x23, x24, LSL #2]\n"
- "mov z26.d, z16.d\n"
"addvl x23, x23, #1\n"
- "fmin z25.s, p3/M, z25.s, z17.s\n"
+ "fmax z23.s, p3/M, z23.s, z18.s\n"
"st1w { z25.s }, p0, [x22]\n"
- "mov z25.d, z16.d\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
- "st1w { z24.s }, p0, [x22, x15, LSL #2]\n"
- "mov z24.d, z16.d\n"
- "fmax z23.s, p3/M, z23.s, z18.s\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
+ "st1w { z24.s }, p0, [x22, x13, LSL #2]\n"
"st1w { z23.s }, p0, [x22, x24, LSL #2]\n"
- "mov z23.d, z16.d\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x10, x25]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "prfm pldl1keep, [x12, x7]\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x16, x7]\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x16, x26]\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x12, x26]\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x13, x5]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x13, x25]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x5]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x11, x17]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "add x21, x6, #0x1\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
+ "add x7, x7, #0x1\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "cmp x7, x19\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "csel x7, x7, XZR, LT\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "csel x6, x6, x21, LT\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "cmp x6, x20\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n"
"fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x11, x25]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x10, x7]\n"
"fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x13, x7]\n"
"fmla z28.s, p3/M, z2.s, z13.s\n"
- "prfm pldl1keep, [x13, x26]\n"
"fmla z27.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x26]\n"
"fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n"
"fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x7]\n"
"fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x16, x17]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x11, x26]\n"
"fmla z27.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x12, x5]\n"
"fmla z25.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x12, x25]\n"
"fmla z24.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13]\n"
+ "ld1w { z11.s }, p2/Z, [x11]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x17]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "ldr x3, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "add x21, x3, #0x1\n"
- "fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11]\n"
- "fmla z29.s, p3/M, z7.s, z10.s\n"
- "ldr x4, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "add x4, x4, #0x1\n"
- "fmla z30.s, p3/M, z8.s, z10.s\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
- "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
- "cmp x4, x19\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
+ "fmla z30.s, p3/M, z2.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x9]\n"
+ "fmla z29.s, p3/M, z7.s, z10.s\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
- "csel x4, x4, XZR, LT\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n"
- "csel x3, x3, x21, LT\n"
+ "fmla z30.s, p3/M, z8.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "cmp x3, x20\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
@@ -459,36 +381,36 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"fmla z26.s, p3/M, z8.s, z11.s\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12]\n"
+ "ld1w { z12.s }, p2/Z, [x10]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z13.s\n"
"fmla z26.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z5.s, z13.s\n"
"fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
@@ -502,13 +424,13 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"fmax z30.s, p3/M, z30.s, z18.s\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x14]\n"
+ "st1w { z31.s }, p0, [x12]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x14, x15, LSL #2]\n"
+ "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
- "st1w { z29.s }, p0, [x14, x24, LSL #2]\n"
+ "st1w { z29.s }, p0, [x12, x24, LSL #2]\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
"fmax z24.s, p3/M, z24.s, z18.s\n"
@@ -516,23 +438,23 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(
"st1w { z28.s }, p0, [x23]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
- "st1w { z27.s }, p0, [x23, x15, LSL #2]\n"
+ "st1w { z27.s }, p0, [x23, x13, LSL #2]\n"
"fmin z25.s, p3/M, z25.s, z17.s\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
"st1w { z26.s }, p0, [x23, x24, LSL #2]\n"
"fmax z23.s, p3/M, z23.s, z18.s\n"
"st1w { z25.s }, p0, [x22]\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
- "st1w { z24.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z24.s }, p0, [x22, x13, LSL #2]\n"
"st1w { z23.s }, p0, [x22, x24, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index 3877ae2f03..72b182679d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -87,413 +87,343 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x17, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x14, ALL, MUL #2\n"
- "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"mov x13, #0x0\n"
- "ld1w { z16.s }, p3/Z, [x16]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"cntw x12\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
"sub x11, XZR, x12\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"cmp x12, %x[n_channels]\n"
- "mov z27.d, z16.d\n"
- "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
- "mov z26.d, z16.d\n"
- "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
- "mov z25.d, z16.d\n"
- "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
- "addvl x16, x16, #16\n"
- "mov z24.d, z16.d\n"
- "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
- "mov z23.d, z16.d\n"
- "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
- "addvl x16, x16, #-6\n"
- "ldp x10, x22, [x15, #0x0]\n"
- "ldp x9, x28, [x15, #0x10]\n"
- "ldr x24, [x15, #0x20]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n"
- "prfm pldl1keep, [x10, x14]\n"
- "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n"
- "prfm pldl1keep, [x22, x14]\n"
- "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
- "prfm pldl1keep, [x9, x14]\n"
- "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
- "prfm pldl1keep, [x28, x14]\n"
- "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n"
- "prfm pldl1keep, [x24, x14]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ldr x26, [x14, #0x20]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x27, [x15, #0x28]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
"whilelt p1.s, x12, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x23, [x15, #0x30]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
"incw x11\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x26, [x15, #0x38]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x27, x14]\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "ldr x25, [x15, #0x40]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "ldr x19, [x15, #0x48]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "ldr x24, [x15, #0x50]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z13.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x24, x14]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z28.s, p3/M, z2.s, z13.s\n"
- "ldr x23, [x15, #0x58]\n"
+ "ldr x21, [x16, #0x8]\n"
"fmla z27.s, p3/M, z1.s, z13.s\n"
- "ldr x22, [x15, #0x60]\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x23, x14]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x15, #0x68]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z27.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
"fmla z25.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x15, #0x78]\n"
"fmla z24.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
- "ldr x10, [x15, #0x80]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z10.s\n"
- "ldr x22, [x15, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x10, x14]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x9, [x15, #0x90]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x9, x14]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
- "ldr x28, [x15, #0x98]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
- "ldr x24, [x15, #0xa0]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x28, x14]\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
- "ldr x27, [x15, #0xa8]\n"
"fmla z28.s, p3/M, z8.s, z10.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z24.s, p3/M, z4.s, z10.s\n"
- "ldr x23, [x15, #0xb0]\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x27, x14]\n"
"fmla z26.s, p3/M, z8.s, z11.s\n"
- "ldr x26, [x15, #0xb8]\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "ldr x25, [x15, #0xc0]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ldp x10, x22, [x15, #0x0]\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x25, x14]\n"
- "addvl x14, x14, #1\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
- "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x10, x14]\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
- "ld1w { z10.s }, p1/Z, [x22, x12, LSL #2]\n"
+ "ldr x26, [x14, #0x20]\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x22, x14]\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
- "ldp x9, x28, [x15, #0x10]\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "ldr x24, [x15, #0x20]\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x9, x14]\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x28, x14]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z13.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z26.s, p3/M, z7.s, z13.s\n"
- "ldr x22, [x17, #0x0]\n"
"fmla z24.s, p3/M, z5.s, z13.s\n"
- "ldr x21, [x17, #0x8]\n"
"fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n"
"incw x13\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
- "ldr x20, [x17, #0x10]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
"whilelt p2.s, x13, %x[n_channels]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ldr x19, [x17, #0x18]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p1/Z, [x28, x12, LSL #2]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z29.s, p3/M, z8.s, z11.s\n"
- "ld1w { z16.s }, p3/Z, [x16]\n"
+ "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n"
"fmla z26.s, p3/M, z5.s, z11.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n"
"fmla z23.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x9, x12, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n"
"fmla z25.s, p3/M, z8.s, z13.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n"
"fmla z24.s, p3/M, z7.s, z13.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z23.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p1/Z, [x24, x12, LSL #2]\n"
+ "ld1w { z13.s }, p1/Z, [x26, x12, LSL #2]\n"
"incw x12\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
"cmp x12, %x[n_channels]\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
- "addvl x16, x16, #16\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
- "addvl x16, x16, #-6\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
- "mov z31.d, z16.d\n"
- "ldr x22, [x17, #0x20]\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
+ "ldr x22, [x16, #0x20]\n"
+ "fmax z26.s, p3/M, z26.s, z18.s\n"
"st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
- "mov z30.d, z16.d\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
- "mov z29.d, z16.d\n"
- "ldr x21, [x17, #0x28]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
- "ldr x20, [x17, #0x30]\n"
- "fmax z26.s, p3/M, z26.s, z18.s\n"
- "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
- "mov z28.d, z16.d\n"
- "ldr x19, [x17, #0x38]\n"
+ "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
+ "fmin z26.s, p3/M, z26.s, z17.s\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
+ "ldr x20, [x16, #0x30]\n"
+ "fmax z24.s, p3/M, z24.s, z18.s\n"
+ "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
+ "fmax z23.s, p3/M, z23.s, z18.s\n"
"st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
- "mov z27.d, z16.d\n"
- "ldr x22, [x17, #0x40]\n"
- "fmin z26.s, p3/M, z26.s, z17.s\n"
"st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
- "mov z26.d, z16.d\n"
"fmin z25.s, p3/M, z25.s, z17.s\n"
- "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
- "mov z25.d, z16.d\n"
- "fmax z24.s, p3/M, z24.s, z18.s\n"
- "fmax z23.s, p3/M, z23.s, z18.s\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
- "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
- "mov z24.d, z16.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
+ "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
+ "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
"st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
- "mov z23.d, z16.d\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x27, [x15, #0x28]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
"incw x11\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x23, [x15, #0x30]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x26, [x15, #0x38]\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x27, x14]\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "ldr x25, [x15, #0x40]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "ldr x19, [x15, #0x48]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "ldr x24, [x15, #0x50]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z13.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x24, x14]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z28.s, p3/M, z2.s, z13.s\n"
- "ldr x23, [x15, #0x58]\n"
+ "ldr x21, [x16, #0x8]\n"
"fmla z27.s, p3/M, z1.s, z13.s\n"
- "ldr x22, [x15, #0x60]\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x23, x14]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x15, #0x68]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z27.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x15, #0x70]\n"
"fmla z25.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x15, #0x78]\n"
"fmla z24.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
- "ldr x10, [x15, #0x80]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z10.s\n"
- "ldr x22, [x15, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x10, x14]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x9, [x15, #0x90]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x9, x14]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
- "ldr x28, [x15, #0x98]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
- "ldr x24, [x15, #0xa0]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x28, x14]\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
- "ldr x27, [x15, #0xa8]\n"
"fmla z28.s, p3/M, z8.s, z10.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z24.s, p3/M, z4.s, z10.s\n"
- "ldr x23, [x15, #0xb0]\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x27, x14]\n"
"fmla z26.s, p3/M, z8.s, z11.s\n"
- "ldr x26, [x15, #0xb8]\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "ldr x25, [x15, #0xc0]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ldr x22, [x17, #0x0]\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x25, x14]\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
- "ldr x21, [x17, #0x8]\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
- "ldr x20, [x17, #0x10]\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
- "ldr x19, [x17, #0x18]\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z13.s\n"
"fmla z26.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z5.s, z13.s\n"
"fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
@@ -510,21 +440,21 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
"st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "ldr x22, [x17, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
"st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
"st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "ldr x21, [x17, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
- "ldr x20, [x17, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.s, p3/M, z24.s, z18.s\n"
"st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
- "ldr x19, [x17, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z25.s, p3/M, z25.s, z17.s\n"
"st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
@@ -533,15 +463,15 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(
"st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
"st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
- "ldr x22, [x17, #0x40]\n"
+ "ldr x22, [x16, #0x40]\n"
"st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index fc9588cd58..d7be9b122e 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst
+class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index 7df8e481c0..84b4b3b72b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -114,45 +114,29 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x14, x15, x22, LSL #2\n"
"ld1w { z13.s }, p3/Z, [x4]\n"
- "mov z31.d, z13.d\n"
- "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n"
"add x13, x14, x22, LSL #2\n"
- "mov z30.d, z13.d\n"
- "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n"
"add x12, x13, x22, LSL #2\n"
- "mov z29.d, z13.d\n"
- "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n"
"add x11, x12, x22, LSL #2\n"
- "mov z28.d, z13.d\n"
- "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n"
"add x10, x7, x7\n"
- "mov z27.d, z13.d\n"
- "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n"
"add x9, x10, x7\n"
- "mov z26.d, z13.d\n"
- "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n"
"add x28, x9, x7\n"
- "mov z25.d, z13.d\n"
- "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n"
"add x27, x28, x7\n"
- "mov z24.d, z13.d\n"
+ "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n"
"mul x19, x2, x20\n" // offset = tile_i * ld_output_row
- "mov z23.d, z13.d\n"
+ "add x26, x17, x17\n"
"madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col
- "mov z22.d, z13.d\n"
"mul x19, x19, x23\n" // offset *= output_tile_size
- "mov z21.d, z13.d\n"
"add x16, x16, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "mov z20.d, z13.d\n"
- "add x26, x16, x20, LSL #2\n"
- "mov z19.d, z13.d\n"
- "add x25, x26, x20, LSL #2\n"
- "mov z18.d, z13.d\n"
+ "add x25, x16, x20, LSL #2\n"
"add x24, x25, x20, LSL #2\n"
- "mov z17.d, z13.d\n"
- "add x23, x17, x17\n"
- "mov z16.d, z13.d\n"
- "add x22, x23, x17\n"
+ "add x23, x24, x20, LSL #2\n"
+ "add x22, x26, x17\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
"ld1w { z9.s }, p2/Z, [x14, x10, LSL #2]\n"
"ld1w { z10.s }, p2/Z, [x8]\n"
@@ -165,39 +149,38 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"ld1w { z12.s }, p2/Z, [x14, x9, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ld1w { z13.s }, p3/Z, [x4]\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
"whilelt p1.s, x6, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
"incw x21\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
"mov p0.b, p2.b\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
"incw x5\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
"incw x6\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x11]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
@@ -206,10 +189,11 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmla z22.s, p3/M, z4.s, z9.s\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z13.s }, p3/Z, [x4]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x15, x27, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
@@ -375,109 +359,93 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"addvl x4, x4, #-6\n"
"fmin z31.s, p3/M, z31.s, z14.s\n"
"st1w { z31.s }, p0, [x16]\n"
- "mov z31.d, z13.d\n"
"fmin z30.s, p3/M, z30.s, z14.s\n"
- "st1w { z30.s }, p0, [x16, x17, LSL #2]\n"
- "mov z30.d, z13.d\n"
"fmin z29.s, p3/M, z29.s, z14.s\n"
- "st1w { z29.s }, p0, [x16, x23, LSL #2]\n"
- "mov z29.d, z13.d\n"
+ "st1w { z30.s }, p0, [x16, x17, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z15.s\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
+ "st1w { z29.s }, p0, [x16, x26, LSL #2]\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
+ "fmax z24.s, p3/M, z24.s, z15.s\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
"st1w { z28.s }, p0, [x16, x22, LSL #2]\n"
- "mov z28.d, z13.d\n"
- "addvl x16, x16, #1\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
- "st1w { z27.s }, p0, [x26]\n"
- "mov z27.d, z13.d\n"
+ "addvl x16, x16, #1\n"
"fmin z26.s, p3/M, z26.s, z14.s\n"
- "st1w { z26.s }, p0, [x26, x17, LSL #2]\n"
- "mov z26.d, z13.d\n"
+ "st1w { z27.s }, p0, [x25]\n"
"fmin z25.s, p3/M, z25.s, z14.s\n"
- "st1w { z25.s }, p0, [x26, x23, LSL #2]\n"
- "mov z25.d, z13.d\n"
- "fmax z24.s, p3/M, z24.s, z15.s\n"
+ "fmin z24.s, p3/M, z24.s, z14.s\n"
+ "st1w { z26.s }, p0, [x25, x17, LSL #2]\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
+ "st1w { z25.s }, p0, [x25, x26, LSL #2]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
- "fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z24.s }, p0, [x26, x22, LSL #2]\n"
- "mov z24.d, z13.d\n"
- "addvl x26, x26, #1\n"
+ "st1w { z24.s }, p0, [x25, x22, LSL #2]\n"
+ "addvl x25, x25, #1\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
- "st1w { z23.s }, p0, [x25]\n"
- "mov z23.d, z13.d\n"
+ "st1w { z23.s }, p0, [x24]\n"
"fmin z22.s, p3/M, z22.s, z14.s\n"
- "st1w { z22.s }, p0, [x25, x17, LSL #2]\n"
- "mov z22.d, z13.d\n"
"fmin z21.s, p3/M, z21.s, z14.s\n"
- "st1w { z21.s }, p0, [x25, x23, LSL #2]\n"
- "mov z21.d, z13.d\n"
+ "st1w { z22.s }, p0, [x24, x17, LSL #2]\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
+ "st1w { z21.s }, p0, [x24, x26, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
+ "fmax z16.s, p3/M, z16.s, z15.s\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "st1w { z20.s }, p0, [x25, x22, LSL #2]\n"
- "mov z20.d, z13.d\n"
- "addvl x25, x25, #1\n"
+ "st1w { z20.s }, p0, [x24, x22, LSL #2]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "st1w { z19.s }, p0, [x24]\n"
- "mov z19.d, z13.d\n"
+ "addvl x24, x24, #1\n"
"fmin z18.s, p3/M, z18.s, z14.s\n"
- "st1w { z18.s }, p0, [x24, x17, LSL #2]\n"
- "mov z18.d, z13.d\n"
+ "st1w { z19.s }, p0, [x23]\n"
"fmin z17.s, p3/M, z17.s, z14.s\n"
- "st1w { z17.s }, p0, [x24, x23, LSL #2]\n"
- "mov z17.d, z13.d\n"
- "fmax z16.s, p3/M, z16.s, z15.s\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z16.s }, p0, [x24, x22, LSL #2]\n"
- "mov z16.d, z13.d\n"
- "addvl x24, x24, #1\n"
+ "st1w { z18.s }, p0, [x23, x17, LSL #2]\n"
+ "st1w { z17.s }, p0, [x23, x26, LSL #2]\n"
+ "st1w { z16.s }, p0, [x23, x22, LSL #2]\n"
+ "addvl x23, x23, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
"ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
"ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x2, #0x1\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x3, x3, #0x1\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
"cmp x3, x19\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
"csel x3, x3, XZR, LT\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
"csel x2, x2, x21, LT\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n"
"cmp x2, x20\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x11]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
@@ -486,8 +454,8 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"fmla z22.s, p3/M, z4.s, z9.s\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
@@ -638,43 +606,43 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
"st1w { z30.s }, p0, [x16, x17, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
- "st1w { z29.s }, p0, [x16, x23, LSL #2]\n"
+ "st1w { z29.s }, p0, [x16, x26, LSL #2]\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
"st1w { z28.s }, p0, [x16, x22, LSL #2]\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
- "st1w { z27.s }, p0, [x26]\n"
+ "st1w { z27.s }, p0, [x25]\n"
"fmin z26.s, p3/M, z26.s, z14.s\n"
"fmin z25.s, p3/M, z25.s, z14.s\n"
- "st1w { z26.s }, p0, [x26, x17, LSL #2]\n"
+ "st1w { z26.s }, p0, [x25, x17, LSL #2]\n"
"fmax z24.s, p3/M, z24.s, z15.s\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
- "st1w { z25.s }, p0, [x26, x23, LSL #2]\n"
+ "st1w { z25.s }, p0, [x25, x26, LSL #2]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
"fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z24.s }, p0, [x26, x22, LSL #2]\n"
+ "st1w { z24.s }, p0, [x25, x22, LSL #2]\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
"fmin z22.s, p3/M, z22.s, z14.s\n"
- "st1w { z23.s }, p0, [x25]\n"
+ "st1w { z23.s }, p0, [x24]\n"
"fmin z21.s, p3/M, z21.s, z14.s\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "st1w { z22.s }, p0, [x25, x17, LSL #2]\n"
+ "st1w { z22.s }, p0, [x24, x17, LSL #2]\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
- "st1w { z21.s }, p0, [x25, x23, LSL #2]\n"
+ "st1w { z21.s }, p0, [x24, x26, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
- "st1w { z20.s }, p0, [x25, x22, LSL #2]\n"
+ "st1w { z20.s }, p0, [x24, x22, LSL #2]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "st1w { z19.s }, p0, [x24]\n"
+ "st1w { z19.s }, p0, [x23]\n"
"fmin z18.s, p3/M, z18.s, z14.s\n"
"fmin z17.s, p3/M, z17.s, z14.s\n"
- "st1w { z18.s }, p0, [x24, x17, LSL #2]\n"
+ "st1w { z18.s }, p0, [x23, x17, LSL #2]\n"
"fmax z16.s, p3/M, z16.s, z15.s\n"
- "st1w { z17.s }, p0, [x24, x23, LSL #2]\n"
+ "st1w { z17.s }, p0, [x23, x26, LSL #2]\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z16.s }, p0, [x24, x22, LSL #2]\n"
+ "st1w { z16.s }, p0, [x23, x22, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
@@ -685,4 +653,4 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 22e12a7b9a..158d44046c 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -98,644 +98,538 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x5, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x7, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x8, ALL, MUL #2\n"
+ "mov x13, #0x0\n"
"ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "mov x17, #0x0\n"
- "ld1w { z13.s }, p3/Z, [x6]\n"
- "mov z31.d, z13.d\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
- "cntw x16\n"
- "mov z30.d, z13.d\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
- "sub x15, XZR, x16\n"
- "mov z29.d, z13.d\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
+ "cntw x12\n"
+ "ld1w { z13.s }, p3/Z, [x15]\n"
+ "sub x11, XZR, x12\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z28.d, z13.d\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
- "cmp x16, %x[n_channels]\n"
- "mov z27.d, z13.d\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
- "mov z26.d, z13.d\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
- "mov z25.d, z13.d\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
- "addvl x6, x6, #16\n"
- "mov z24.d, z13.d\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "mov z23.d, z13.d\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
- "mov z22.d, z13.d\n"
- "ldp x14, x13, [x7, #0x0]\n"
- "mov z21.d, z13.d\n"
- "ldp x12, x11, [x7, #0x10]\n"
- "mov z20.d, z13.d\n"
- "ld1w { z9.s }, p2/Z, [x14, x17, LSL #2]\n"
- "mov z19.d, z13.d\n"
- "mov z18.d, z13.d\n"
- "prfm pldl1keep, [x14, x8]\n"
- "mov z17.d, z13.d\n"
- "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n"
- "mov z16.d, z13.d\n"
- "prfm pldl1keep, [x13, x8]\n"
- "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n"
- "prfm pldl1keep, [x12, x8]\n"
- "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n"
- "prfm pldl1keep, [x11, x8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x7, #0x20]\n"
- "whilelt p1.s, x16, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x21, [x7, #0x28]\n"
- "incw x15\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x20, [x7, #0x30]\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x20]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
+ "incw x11\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x20, x8]\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
- "ldr x19, [x7, #0x38]\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
- "ldr x10, [x7, #0x40]\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x19, x8]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
- "ldr x9, [x7, #0x48]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ldr x28, [x7, #0x50]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "ldr x27, [x7, #0x58]\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x27, x8]\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
- "ldr x26, [x7, #0x60]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
- "ldr x25, [x7, #0x68]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.s, p3/M, z6.s, z9.s\n"
- "ldr x24, [x7, #0x70]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x26, x8]\n"
"fmla z22.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x24, x8]\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x7, #0x78]\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "ldr x14, [x7, #0x80]\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x14, x8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z26.s, p3/M, z8.s, z10.s\n"
- "ldr x13, [x7, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.s, p3/M, z7.s, z10.s\n"
- "ldr x12, [x7, #0x90]\n"
+ "ld1w { z13.s }, p3/Z, [x15]\n"
"fmla z24.s, p3/M, z6.s, z10.s\n"
- "ldr x11, [x7, #0x98]\n"
"fmla z22.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
"fmla z21.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z20.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x11, x8]\n"
"fmla z18.s, p3/M, z2.s, z10.s\n"
- "ldr x22, [x7, #0xa0]\n"
"fmla z17.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x7, #0xa8]\n"
"fmla z16.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
"fmla z28.s, p3/M, z5.s, z12.s\n"
- "ldr x20, [x7, #0xb0]\n"
"fmla z24.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z23.s, p3/M, z6.s, z11.s\n"
- "ldr x19, [x7, #0xb8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x20, x8]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x19, x8]\n"
"fmla z29.s, p3/M, z3.s, z10.s\n"
- "ldr x10, [x7, #0xc0]\n"
"fmla z27.s, p3/M, z2.s, z10.s\n"
- "ldr x9, [x7, #0xc8]\n"
"fmla z26.s, p3/M, z1.s, z10.s\n"
- "ldr x28, [x7, #0xd0]\n"
"fmla z25.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z20.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ldr x27, [x7, #0xd8]\n"
"fmla z26.s, p3/M, z2.s, z12.s\n"
- "ldr x26, [x7, #0xe0]\n"
"fmla z25.s, p3/M, z1.s, z12.s\n"
- "ldr x25, [x7, #0xe8]\n"
"fmla z24.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z19.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x27, x8]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z10.s\n"
- "prfm pldl1keep, [x26, x8]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z27.s, p3/M, z4.s, z10.s\n"
- "ldr x24, [x7, #0xf0]\n"
"fmla z26.s, p3/M, z3.s, z10.s\n"
- "ldr x23, [x7, #0xf8]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x14, [x7, #0x100]\n"
"fmla z22.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z17.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x24, x8]\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.s, p3/M, z7.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x14, x8]\n"
"fmla z25.s, p3/M, z5.s, z12.s\n"
- "ldr x13, [x7, #0x108]\n"
"fmla z24.s, p3/M, z4.s, z12.s\n"
- "ldr x12, [x7, #0x110]\n"
"fmla z21.s, p3/M, z2.s, z12.s\n"
- "ldr x11, [x7, #0x118]\n"
"fmla z20.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z29.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x8]\n"
- "addvl x8, x8, #1\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.s, p3/M, z6.s, z11.s\n"
- "ldr x22, [x5, #0x0]\n"
"fmla z23.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x5, #0x8]\n"
"fmla z22.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x5, #0x10]\n"
"fmla z19.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x5, #0x18]\n"
"fmla z18.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z13.s }, p3/Z, [x6]\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
"fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z10.s\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.s, p3/M, z3.s, z10.s\n"
"fmla z23.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z25.s, p3/M, z8.s, z11.s\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z5.s, z11.s\n"
"fmla z20.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z2.s, z11.s\n"
"fmla z16.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z28.s, p3/M, z8.s, z12.s\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.s, p3/M, z5.s, z12.s\n"
"fmla z20.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z27.s, p3/M, z6.s, z10.s\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
"fmla z19.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z22.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z6.s, z11.s\n"
"fmla z23.s, p3/M, z8.s, z11.s\n"
"fmla z19.s, p3/M, z5.s, z11.s\n"
"fmla z18.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z12.s\n"
"fmla z20.s, p3/M, z5.s, z12.s\n"
"fmla z16.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z19.s, p3/M, z8.s, z10.s\n"
"fmla z18.s, p3/M, z7.s, z10.s\n"
"fmla z17.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z22.s, p3/M, z8.s, z11.s\n"
"fmla z21.s, p3/M, z7.s, z11.s\n"
"fmla z20.s, p3/M, z6.s, z11.s\n"
"fmla z18.s, p3/M, z5.s, z11.s\n"
"fmla z17.s, p3/M, z4.s, z11.s\n"
"fmla z16.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z10.s\n"
- "ldp x14, x13, [x7, #0x0]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
"fmla z18.s, p3/M, z8.s, z12.s\n"
- "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n"
"fmla z17.s, p3/M, z7.s, z12.s\n"
"fmla z16.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x14, x8]\n"
"fmla z27.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
"fmla z26.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n"
- "incw x17\n"
+ "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z29.s, p3/M, z5.s, z11.s\n"
- "ldp x12, x11, [x7, #0x10]\n"
- "whilelt p2.s, x17, %x[n_channels]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z25.s, p3/M, z2.s, z11.s\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z24.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x12, x16, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n"
"fmla z23.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x12, x8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z22.s, p3/M, z6.s, z12.s\n"
- "prfm pldl1keep, [x11, x8]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z19.s, p3/M, z4.s, z12.s\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
"fmla z18.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p1/Z, [x11, x16, LSL #2]\n"
+ "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n"
"fmla z21.s, p3/M, z8.s, z10.s\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z20.s, p3/M, z7.s, z10.s\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
"fmla z17.s, p3/M, z5.s, z10.s\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z16.s, p3/M, z4.s, z10.s\n"
- "ld1w { z10.s }, p1/Z, [x13, x16, LSL #2]\n"
- "incw x16\n"
+ "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n"
+ "incw x12\n"
"fmax z31.s, p3/M, z31.s, z15.s\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
- "addvl x6, x6, #16\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z30.s, p3/M, z30.s, z15.s\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "cmp x16, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.s, p3/M, z29.s, z15.s\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.s, p3/M, z28.s, z15.s\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
"fmin z31.s, p3/M, z31.s, z14.s\n"
- "st1w { z31.s }, p0, [x22, x15, LSL #2]\n"
- "mov z31.d, z13.d\n"
+ "st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z14.s\n"
- "ldr x22, [x5, #0x20]\n"
"fmin z29.s, p3/M, z29.s, z14.s\n"
- "st1w { z30.s }, p0, [x21, x15, LSL #2]\n"
- "mov z30.d, z13.d\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
- "st1w { z29.s }, p0, [x20, x15, LSL #2]\n"
- "mov z29.d, z13.d\n"
- "ldr x21, [x5, #0x28]\n"
+ "st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
- "ldr x20, [x5, #0x30]\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
- "st1w { z28.s }, p0, [x19, x15, LSL #2]\n"
- "mov z28.d, z13.d\n"
- "ldr x19, [x5, #0x38]\n"
+ "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
- "st1w { z27.s }, p0, [x22, x15, LSL #2]\n"
- "mov z27.d, z13.d\n"
- "ldr x22, [x5, #0x40]\n"
- "fmin z26.s, p3/M, z26.s, z14.s\n"
- "st1w { z26.s }, p0, [x21, x15, LSL #2]\n"
- "mov z26.d, z13.d\n"
- "fmin z25.s, p3/M, z25.s, z14.s\n"
- "ldr x21, [x5, #0x48]\n"
+ "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmax z24.s, p3/M, z24.s, z15.s\n"
- "st1w { z25.s }, p0, [x20, x15, LSL #2]\n"
- "mov z25.d, z13.d\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
- "ldr x20, [x5, #0x50]\n"
+ "st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
+ "fmin z26.s, p3/M, z26.s, z14.s\n"
+ "ldr x20, [x16, #0x30]\n"
+ "fmin z25.s, p3/M, z25.s, z14.s\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z24.s }, p0, [x19, x15, LSL #2]\n"
- "mov z24.d, z13.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
- "ldr x19, [x5, #0x58]\n"
+ "st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
- "st1w { z23.s }, p0, [x22, x15, LSL #2]\n"
- "mov z23.d, z13.d\n"
+ "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
- "ldr x22, [x5, #0x60]\n"
- "fmin z22.s, p3/M, z22.s, z14.s\n"
- "st1w { z22.s }, p0, [x21, x15, LSL #2]\n"
- "mov z22.d, z13.d\n"
- "fmin z21.s, p3/M, z21.s, z14.s\n"
- "ldr x21, [x5, #0x68]\n"
+ "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
- "st1w { z21.s }, p0, [x20, x15, LSL #2]\n"
- "mov z21.d, z13.d\n"
+ "st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
- "ldr x20, [x5, #0x70]\n"
+ "ldr x21, [x16, #0x48]\n"
+ "fmin z22.s, p3/M, z22.s, z14.s\n"
+ "ldr x20, [x16, #0x50]\n"
+ "fmin z21.s, p3/M, z21.s, z14.s\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "st1w { z20.s }, p0, [x19, x15, LSL #2]\n"
- "mov z20.d, z13.d\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "ldr x19, [x5, #0x78]\n"
+ "st1w { z22.s }, p0, [x21, x11, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
- "st1w { z19.s }, p0, [x22, x15, LSL #2]\n"
- "mov z19.d, z13.d\n"
+ "st1w { z21.s }, p0, [x20, x11, LSL #2]\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
+ "st1w { z20.s }, p0, [x19, x11, LSL #2]\n"
+ "fmax z16.s, p3/M, z16.s, z15.s\n"
+ "st1w { z19.s }, p0, [x22, x11, LSL #2]\n"
+ "ldr x21, [x16, #0x68]\n"
"fmin z18.s, p3/M, z18.s, z14.s\n"
- "st1w { z18.s }, p0, [x21, x15, LSL #2]\n"
- "mov z18.d, z13.d\n"
+ "ldr x20, [x16, #0x70]\n"
"fmin z17.s, p3/M, z17.s, z14.s\n"
- "st1w { z17.s }, p0, [x20, x15, LSL #2]\n"
- "mov z17.d, z13.d\n"
- "fmax z16.s, p3/M, z16.s, z15.s\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z16.s }, p0, [x19, x15, LSL #2]\n"
- "mov z16.d, z13.d\n"
+ "st1w { z18.s }, p0, [x21, x11, LSL #2]\n"
+ "st1w { z17.s }, p0, [x20, x11, LSL #2]\n"
+ "st1w { z16.s }, p0, [x19, x11, LSL #2]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x7, #0x20]\n"
- "incw x15\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x21, [x7, #0x28]\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x20]\n"
+ "incw x11\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x20, [x7, #0x30]\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x20, x8]\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
- "ldr x19, [x7, #0x38]\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
- "ldr x10, [x7, #0x40]\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x19, x8]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
- "ldr x9, [x7, #0x48]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ldr x28, [x7, #0x50]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "ldr x27, [x7, #0x58]\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x27, x8]\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
- "ldr x26, [x7, #0x60]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
- "ldr x25, [x7, #0x68]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.s, p3/M, z6.s, z9.s\n"
- "ldr x24, [x7, #0x70]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x26, x8]\n"
"fmla z22.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x24, x8]\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x7, #0x78]\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "ldr x14, [x7, #0x80]\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x14, x8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z26.s, p3/M, z8.s, z10.s\n"
- "ldr x13, [x7, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.s, p3/M, z7.s, z10.s\n"
- "ldr x12, [x7, #0x90]\n"
"fmla z24.s, p3/M, z6.s, z10.s\n"
- "ldr x11, [x7, #0x98]\n"
"fmla z22.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
"fmla z21.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z20.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x11, x8]\n"
"fmla z18.s, p3/M, z2.s, z10.s\n"
- "ldr x22, [x7, #0xa0]\n"
"fmla z17.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x7, #0xa8]\n"
"fmla z16.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
"fmla z28.s, p3/M, z5.s, z12.s\n"
- "ldr x20, [x7, #0xb0]\n"
"fmla z24.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z23.s, p3/M, z6.s, z11.s\n"
- "ldr x19, [x7, #0xb8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x20, x8]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x19, x8]\n"
"fmla z29.s, p3/M, z3.s, z10.s\n"
- "ldr x10, [x7, #0xc0]\n"
"fmla z27.s, p3/M, z2.s, z10.s\n"
- "ldr x9, [x7, #0xc8]\n"
"fmla z26.s, p3/M, z1.s, z10.s\n"
- "ldr x28, [x7, #0xd0]\n"
"fmla z25.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z20.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ldr x27, [x7, #0xd8]\n"
"fmla z26.s, p3/M, z2.s, z12.s\n"
- "ldr x26, [x7, #0xe0]\n"
"fmla z25.s, p3/M, z1.s, z12.s\n"
- "ldr x25, [x7, #0xe8]\n"
"fmla z24.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z19.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x27, x8]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z10.s\n"
- "prfm pldl1keep, [x26, x8]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z27.s, p3/M, z4.s, z10.s\n"
- "ldr x24, [x7, #0xf0]\n"
"fmla z26.s, p3/M, z3.s, z10.s\n"
- "ldr x23, [x7, #0xf8]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x14, [x7, #0x100]\n"
"fmla z22.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z17.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x24, x8]\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.s, p3/M, z7.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x14, x8]\n"
"fmla z25.s, p3/M, z5.s, z12.s\n"
- "ldr x13, [x7, #0x108]\n"
"fmla z24.s, p3/M, z4.s, z12.s\n"
- "ldr x12, [x7, #0x110]\n"
"fmla z21.s, p3/M, z2.s, z12.s\n"
- "ldr x11, [x7, #0x118]\n"
"fmla z20.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z29.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x8]\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.s, p3/M, z6.s, z11.s\n"
- "ldr x22, [x5, #0x0]\n"
"fmla z23.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x5, #0x8]\n"
"fmla z22.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x5, #0x10]\n"
"fmla z19.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x5, #0x18]\n"
"fmla z18.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
"fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z10.s\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.s, p3/M, z3.s, z10.s\n"
"fmla z23.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z25.s, p3/M, z8.s, z11.s\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z5.s, z11.s\n"
"fmla z20.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z2.s, z11.s\n"
"fmla z16.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z28.s, p3/M, z8.s, z12.s\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.s, p3/M, z5.s, z12.s\n"
"fmla z20.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z27.s, p3/M, z6.s, z10.s\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
"fmla z19.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z22.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z6.s, z11.s\n"
"fmla z23.s, p3/M, z8.s, z11.s\n"
"fmla z19.s, p3/M, z5.s, z11.s\n"
"fmla z18.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z12.s\n"
"fmla z20.s, p3/M, z5.s, z12.s\n"
"fmla z16.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z19.s, p3/M, z8.s, z10.s\n"
"fmla z18.s, p3/M, z7.s, z10.s\n"
"fmla z17.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z22.s, p3/M, z8.s, z11.s\n"
"fmla z21.s, p3/M, z7.s, z11.s\n"
"fmla z20.s, p3/M, z6.s, z11.s\n"
"fmla z18.s, p3/M, z5.s, z11.s\n"
"fmla z17.s, p3/M, z4.s, z11.s\n"
"fmla z16.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z10.s\n"
"fmla z18.s, p3/M, z8.s, z12.s\n"
"fmla z17.s, p3/M, z7.s, z12.s\n"
"fmla z16.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z10.s\n"
"fmla z27.s, p3/M, z1.s, z10.s\n"
"fmla z26.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z11.s\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
"fmla z25.s, p3/M, z2.s, z11.s\n"
@@ -753,68 +647,68 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(
"fmax z29.s, p3/M, z29.s, z15.s\n"
"fmax z28.s, p3/M, z28.s, z15.s\n"
"fmin z31.s, p3/M, z31.s, z14.s\n"
- "st1w { z31.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z14.s\n"
"fmin z29.s, p3/M, z29.s, z14.s\n"
- "ldr x22, [x5, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
- "st1w { z30.s }, p0, [x21, x15, LSL #2]\n"
+ "st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
- "st1w { z29.s }, p0, [x20, x15, LSL #2]\n"
+ "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
- "st1w { z28.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmax z24.s, p3/M, z24.s, z15.s\n"
- "ldr x21, [x5, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
- "ldr x20, [x5, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
- "ldr x19, [x5, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z26.s, p3/M, z26.s, z14.s\n"
- "st1w { z27.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
"fmin z25.s, p3/M, z25.s, z14.s\n"
"fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z26.s }, p0, [x21, x15, LSL #2]\n"
+ "st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
- "ldr x22, [x5, #0x40]\n"
+ "ldr x22, [x16, #0x40]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
- "ldr x21, [x5, #0x48]\n"
+ "ldr x21, [x16, #0x48]\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
- "st1w { z25.s }, p0, [x20, x15, LSL #2]\n"
+ "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
- "st1w { z24.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
- "st1w { z23.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
"fmin z22.s, p3/M, z22.s, z14.s\n"
- "ldr x20, [x5, #0x50]\n"
+ "ldr x20, [x16, #0x50]\n"
"fmin z21.s, p3/M, z21.s, z14.s\n"
- "ldr x19, [x5, #0x58]\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "ldr x22, [x5, #0x60]\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "st1w { z22.s }, p0, [x21, x15, LSL #2]\n"
+ "st1w { z22.s }, p0, [x21, x11, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
- "st1w { z21.s }, p0, [x20, x15, LSL #2]\n"
+ "st1w { z21.s }, p0, [x20, x11, LSL #2]\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
- "st1w { z20.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z20.s }, p0, [x19, x11, LSL #2]\n"
"fmax z16.s, p3/M, z16.s, z15.s\n"
- "st1w { z19.s }, p0, [x22, x15, LSL #2]\n"
- "ldr x21, [x5, #0x68]\n"
+ "st1w { z19.s }, p0, [x22, x11, LSL #2]\n"
+ "ldr x21, [x16, #0x68]\n"
"fmin z18.s, p3/M, z18.s, z14.s\n"
- "ldr x20, [x5, #0x70]\n"
+ "ldr x20, [x16, #0x70]\n"
"fmin z17.s, p3/M, z17.s, z14.s\n"
- "ldr x19, [x5, #0x78]\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z18.s }, p0, [x21, x15, LSL #2]\n"
- "st1w { z17.s }, p0, [x20, x15, LSL #2]\n"
- "st1w { z16.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z18.s }, p0, [x21, x11, LSL #2]\n"
+ "st1w { z17.s }, p0, [x20, x11, LSL #2]\n"
+ "st1w { z16.s }, p0, [x19, x11, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index 2119c06965..28d44d0b5f 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst
+class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 78e67e1be1..ac33dcbce5 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -88,318 +88,250 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(
__asm__ __volatile__(
"ptrue p3.b\n"
- "mov x5, #0x0\n"
- "mov x6, #0x0\n"
+ "mov x7, #0x0\n"
+ "mov x8, #0x0\n"
"1:" // Tile loop
- "str x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "mov x21, #0x4\n"
- "str x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "cntb x7\n"
- "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x7, x7, XZR, LSL #4\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
- "cntb x17\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
- "cntb x15\n"
- "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n"
- "mul x19, x5, x20\n" // offset = tile_i * ld_input_row
- "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
- "madd x19, x6, x16, x19\n" // offset += tile_j * ld_input_col
- "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
- "mul x19, x19, x21\n" // offset *= kernel_stride * output_size
- "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n"
- "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
+ "str x7, [%x[params_struct], %[offsetof_args_tile_i]]\n"
+ "mov x23, #0x4\n"
+ "str x8, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "mov x17, #0x2\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
+ "mov x15, #0x0\n"
+ "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
+ "cntw x14\n"
+ "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
+ "sub x12, XZR, x14\n"
+ "ldr x21, [%x[params_struct], %[offsetof_args_inptr]]\n"
+ "mul x19, x7, x22\n" // offset = tile_i * ld_input_row
+ "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
+ "madd x19, x8, x13, x19\n" // offset += tile_j * ld_input_col
+ "ldr x11, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
+ "mul x19, x19, x23\n" // offset *= kernel_stride * output_size
+ "ldr x10, [%x[params_struct], %[offsetof_args_outptr]]\n"
+ "add x21, x21, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
"ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "add x11, x14, x20, LSL #2\n"
+ "add x9, x21, x22, LSL #2\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "add x10, x11, x20, LSL #2\n"
- "ld1w { z17.s }, p3/Z, [x8]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
- "add x9, x10, x20, LSL #2\n"
- "mov z30.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
- "add x28, x9, x20, LSL #2\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
- "add x27, x16, x16\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
- "add x26, x27, x16\n"
- "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
- "add x25, x26, x16\n"
- "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
- "add x17, x17, x16, LSL #4\n"
- "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
- "add x15, x15, x27, LSL #4\n"
- "cntb x24\n"
- "prfm pldl1keep, [x10, x15]\n"
- "prfm pldl1keep, [x14, x7]\n"
- "add x24, x24, x26, LSL #4\n"
- "prfm pldl1keep, [x14, x17]\n"
- "cntb x23\n"
- "prfm pldl1keep, [x14, x24]\n"
- "add x23, x23, x25, LSL #4\n"
- "mov x20, #0x2\n"
- "prfm pldl1keep, [x14, x23]\n"
- "prfm pldl1keep, [x11, x7]\n"
- "mul x19, x5, x22\n" // offset = tile_i * ld_output_row
- "prfm pldl1keep, [x11, x17]\n"
- "madd x19, x6, x13, x19\n" // offset += tile_j * ld_output_col
- "prfm pldl1keep, [x14, x15]\n"
- "mul x19, x19, x20\n" // offset *= output_tile_size
- "mov x21, #0x0\n"
- "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "add x22, x12, x22, LSL #2\n"
- "cntw x20\n"
- "sub x19, XZR, x20\n"
+ "add x28, x9, x22, LSL #2\n"
+ "ld1w { z17.s }, p3/Z, [x16]\n"
+ "add x27, x28, x22, LSL #2\n"
+ "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "add x26, x27, x22, LSL #2\n"
+ "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "add x25, x13, x13\n"
+ "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "add x24, x25, x13\n"
+ "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "add x23, x24, x13\n"
+ "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "mul x19, x7, x20\n" // offset = tile_i * ld_output_row
+ "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col
+ "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "mul x19, x19, x17\n" // offset *= output_tile_size
"whilelt p2.s, XZR, %x[n_channels]\n"
- "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x14]\n"
- "addvl x8, x8, #16\n"
- "ld1w { z11.s }, p2/Z, [x14, x16, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
- "addvl x8, x8, #-6\n"
- "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n"
- "ld1w { z13.s }, p2/Z, [x14, x25, LSL #2]\n"
- "ld1w { z14.s }, p2/Z, [x11]\n"
- "ld1w { z15.s }, p2/Z, [x11, x16, LSL #2]\n"
- "ld1w { z16.s }, p2/Z, [x14, x27, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x28, x25, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x21]\n"
+ "add x10, x10, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "add x22, x10, x20, LSL #2\n"
+ "ld1w { z12.s }, p2/Z, [x21, x24, LSL #2]\n"
+ "addvl x16, x16, #16\n"
+ "ld1w { z13.s }, p2/Z, [x21, x23, LSL #2]\n"
+ "cmp x14, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
+ "addvl x16, x16, #-6\n"
+ "ld1w { z14.s }, p2/Z, [x9]\n"
+ "ld1w { z15.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x21, x25, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x11, x24]\n"
- "whilelt p1.s, x20, %x[n_channels]\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x11, x23]\n"
- "incw x19\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x11, x15]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "whilelt p1.s, x14, %x[n_channels]\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "incw x12\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x9, x7]\n"
- "incw x21\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z17.s }, p3/Z, [x16]\n"
+ "incw x15\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x10, x7]\n"
- "addvl x14, x14, #1\n"
+ "addvl x21, x21, #1\n"
+ "ld1w { z10.s }, p1/Z, [x21]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n"
- "incw x20\n"
+ "incw x14\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
- "prfm pldl1keep, [x9, x17]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x9]\n"
- "addvl x11, x11, #1\n"
+ "ld1w { z14.s }, p2/Z, [x27]\n"
+ "addvl x9, x9, #1\n"
"fmla z30.s, p3/M, z0.s, z16.s\n"
- "prfm pldl1keep, [x10, x17]\n"
- "prfm pldl1keep, [x9, x24]\n"
"fmla z29.s, p3/M, z3.s, z14.s\n"
- "prfm pldl1keep, [x10, x24]\n"
- "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x10]\n"
+ "ld1w { z15.s }, p2/Z, [x28]\n"
"fmla z30.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z15.s\n"
- "prfm pldl1keep, [x9, x23]\n"
- "prfm pldl1keep, [x28, x7]\n"
+ "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
- "prfm pldl1keep, [x10, x23]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n"
+ "addvl x28, x28, #1\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "addvl x10, x10, #1\n"
+ "ld1w { z9.s }, p1/Z, [x28, x25, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
- "prfm pldl1keep, [x28, x17]\n"
+ "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z16.s\n"
- "prfm pldl1keep, [x9, x15]\n"
- "prfm pldl1keep, [x28, x24]\n"
"fmla z31.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26]\n"
"fmla z28.s, p3/M, z4.s, z13.s\n"
- "ld1w { z15.s }, p2/Z, [x28]\n"
+ "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z7.s, z12.s\n"
- "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n"
- "prfm pldl1keep, [x28, x15]\n"
+ "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "fmla z29.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n"
- "addvl x9, x9, #1\n"
+ "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n"
+ "addvl x27, x27, #1\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x28, x23]\n"
- "fmla z29.s, p3/M, z6.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n"
+ "ld1w { z12.s }, p1/Z, [x21, x24, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z11.s\n"
- "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n"
- "prfm pldl1keep, [x10, x15]\n"
+ "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "fmla z29.s, p3/M, z7.s, z13.s\n"
+ "ld1w { z13.s }, p1/Z, [x21, x23, LSL #2]\n"
"fmax z31.s, p3/M, z31.s, z19.s\n"
- "ld1w { z10.s }, p1/Z, [x14]\n"
"fmla z28.s, p3/M, z5.s, z14.s\n"
- "fmla z29.s, p3/M, z7.s, z13.s\n"
- "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n"
"fmax z30.s, p3/M, z30.s, z19.s\n"
- "prfm pldl1keep, [x14, x7]\n"
- "prfm pldl1keep, [x14, x17]\n"
+ "fmla z29.s, p3/M, z5.s, z16.s\n"
+ "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
"fmin z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x14, x26, LSL #2]\n"
+ "st1w { z31.s }, p0, [x10]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "fmla z29.s, p3/M, z5.s, z16.s\n"
- "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n"
- "whilelt p2.s, x21, %x[n_channels]\n"
+ "fmla z29.s, p3/M, z8.s, z15.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n"
+ "whilelt p2.s, x15, %x[n_channels]\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "prfm pldl1keep, [x14, x24]\n"
- "addvl x28, x28, #1\n"
+ "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "addvl x26, x26, #1\n"
"fmla z28.s, p3/M, z3.s, z16.s\n"
- "ld1w { z13.s }, p1/Z, [x14, x25, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "fmla z29.s, p3/M, z8.s, z15.s\n"
- "prfm pldl1keep, [x14, x23]\n"
- "prfm pldl1keep, [x11, x7]\n"
- "fmla z28.s, p3/M, z7.s, z14.s\n"
- "ld1w { z14.s }, p1/Z, [x11]\n"
- "prfm pldl1keep, [x11, x17]\n"
+ "ld1w { z16.s }, p1/Z, [x21, x25, LSL #2]\n"
+ "cmp x14, %x[n_channels]\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
- "ld1w { z16.s }, p1/Z, [x14, x27, LSL #2]\n"
- "fmla z28.s, p3/M, z6.s, z15.s\n"
- "ld1w { z15.s }, p1/Z, [x11, x16, LSL #2]\n"
- "prfm pldl1keep, [x14, x15]\n"
+ "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "st1w { z30.s }, p0, [x10, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z7.s, z14.s\n"
+ "ld1w { z14.s }, p1/Z, [x9]\n"
+ "addvl x10, x10, #1\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
- "st1w { z31.s }, p0, [x12]\n"
+ "st1w { z29.s }, p0, [x22]\n"
+ "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p1/Z, [x9, x13, LSL #2]\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x14, x16, LSL #2]\n"
- "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
- "st1w { z29.s }, p0, [x22]\n"
- "addvl x12, x12, #1\n"
+ "addvl x16, x16, #16\n"
+ "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z17.s }, p3/Z, [x8]\n"
- "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
- "mov z30.d, z17.d\n"
- "st1w { z28.s }, p0, [x22, x13, LSL #2]\n"
+ "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
+ "addvl x16, x16, #-6\n"
+ "st1w { z28.s }, p0, [x22, x11, LSL #2]\n"
"addvl x22, x22, #1\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
- "addvl x8, x8, #16\n"
- "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
- "addvl x8, x8, #-6\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x11, x24]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x11, x23]\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x11, x15]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x9, x7]\n"
- "prfm pldl1keep, [x10, x7]\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "add x21, x7, #0x1\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
+ "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
+ "add x8, x8, #0x1\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x9, x17]\n"
+ "cmp x8, x19\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
- "prfm pldl1keep, [x10, x17]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n"
+ "csel x8, x8, XZR, LT\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n"
+ "csel x7, x7, x21, LT\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x9]\n"
- "prfm pldl1keep, [x9, x24]\n"
+ "ld1w { z14.s }, p2/Z, [x27]\n"
+ "cmp x7, x20\n"
"fmla z30.s, p3/M, z0.s, z16.s\n"
- "prfm pldl1keep, [x10, x24]\n"
"fmla z29.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n"
- "prfm pldl1keep, [x9, x23]\n"
+ "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x10]\n"
+ "ld1w { z15.s }, p2/Z, [x28]\n"
"fmla z30.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z15.s\n"
- "prfm pldl1keep, [x28, x7]\n"
- "prfm pldl1keep, [x10, x23]\n"
"fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
- "prfm pldl1keep, [x28, x17]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x9, x15]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z16.s\n"
- "prfm pldl1keep, [x28, x24]\n"
- "prfm pldl1keep, [x28, x15]\n"
"fmla z31.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26]\n"
"fmla z28.s, p3/M, z4.s, z13.s\n"
- "ld1w { z15.s }, p2/Z, [x28]\n"
+ "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z7.s, z12.s\n"
- "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n"
- "prfm pldl1keep, [x28, x23]\n"
+ "fmla z29.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "add x21, x5, #0x1\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n"
- "fmla z29.s, p3/M, z6.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z11.s\n"
- "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "add x6, x6, #0x1\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z13.s\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
"fmax z31.s, p3/M, z31.s, z19.s\n"
- "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
- "cmp x6, x19\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n"
+ "fmax z30.s, p3/M, z30.s, z19.s\n"
"fmla z29.s, p3/M, z5.s, z16.s\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "csel x6, x6, XZR, LT\n"
"fmin z31.s, p3/M, z31.s, z18.s\n"
- "st1w { z31.s }, p0, [x12]\n"
- "fmla z28.s, p3/M, z7.s, z14.s\n"
- "csel x5, x5, x21, LT\n"
+ "st1w { z31.s }, p0, [x10]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
"fmla z29.s, p3/M, z8.s, z15.s\n"
- "cmp x5, x20\n"
- "fmax z30.s, p3/M, z30.s, z19.s\n"
- "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "st1w { z30.s }, p0, [x10, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
+ "fmla z28.s, p3/M, z7.s, z14.s\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
"st1w { z29.s }, p0, [x22]\n"
+ "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z28.s }, p0, [x22, x13, LSL #2]\n"
+ "st1w { z28.s }, p0, [x22, x11, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index 8555cfea7c..829b0ff2c7 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -89,309 +89,249 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x12, ALL, MUL #2\n"
+ "mov x13, #0x0\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "mov x11, #0x0\n"
- "ldp x10, x9, [x19, #0x0]\n"
- "cntw x28\n"
- "ldp x27, x26, [x19, #0x10]\n"
- "sub x25, XZR, x28\n"
- "ld1w { z17.s }, p3/Z, [x14]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
+ "cntw x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z30.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
- "cmp x28, %x[n_channels]\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
- "addvl x14, x14, #16\n"
- "ldp x21, x20, [x13, #0x0]\n"
- "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
- "addvl x14, x14, #-6\n"
- "ld1w { z9.s }, p2/Z, [x21, x11, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z10.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
- "ldp x24, x23, [x13, #0x10]\n"
- "ldp x22, x21, [x13, #0x20]\n"
- "ldp x20, x19, [x13, #0x30]\n"
- "ld1w { z11.s }, p2/Z, [x24, x11, LSL #2]\n"
- "prfm pldl1keep, [x24, x12]\n"
- "ld1w { z12.s }, p2/Z, [x23, x11, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ld1w { z14.s }, p2/Z, [x21, x11, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z15.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
- "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z17.s }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x19, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x13, #0x40]\n"
- "whilelt p1.s, x28, %x[n_channels]\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "ldr x21, [x13, #0x48]\n"
- "incw x25\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x20, [x13, #0x50]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x40]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "ldr x25, [x14, #0x48]\n"
+ "incw x9\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
+ "ldr x24, [x14, #0x50]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ldr x19, [x13, #0x58]\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.s, p3/M, z0.s, z16.s\n"
- "ldr x21, [x13, #0x60]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.s, p3/M, z3.s, z14.s\n"
- "ldr x20, [x13, #0x68]\n"
- "ldr x19, [x13, #0x70]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z15.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "ldr x19, [x13, #0x78]\n"
- "ldr x21, [x13, #0x80]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "ldr x20, [x13, #0x88]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x24, [x13, #0x90]\n"
+ "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z16.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.s, p3/M, z6.s, z15.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z30.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x24, x12]\n"
- "ldr x23, [x13, #0x98]\n"
+ "ld1w { z17.s }, p3/Z, [x15]\n"
"fmla z29.s, p3/M, z6.s, z15.s\n"
- "ldr x22, [x13, #0xa0]\n"
- "fmax z31.s, p3/M, z31.s, z19.s\n"
- "ldr x21, [x13, #0xa8]\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "fmin z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
+ "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z11.s\n"
- "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "fmax z31.s, p3/M, z31.s, z19.s\n"
+ "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z13.s\n"
- "ldr x20, [x13, #0xb0]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmax z30.s, p3/M, z30.s, z19.s\n"
- "ldr x19, [x13, #0xb8]\n"
- "ldr x22, [x13, #0xc0]\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z29.s, p3/M, z5.s, z16.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
+ "fmin z31.s, p3/M, z31.s, z18.s\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
- "fmla z28.s, p3/M, z7.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
+ "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z15.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "incw x11\n"
- "fmla z28.s, p3/M, z6.s, z15.s\n"
- "ldp x21, x20, [x13, #0x0]\n"
- "whilelt p2.s, x11, %x[n_channels]\n"
+ "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n"
+ "fmla z28.s, p3/M, z7.s, z14.s\n"
+ "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n"
+ "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
- "ldp x24, x23, [x13, #0x10]\n"
- "addvl x12, x12, #1\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z9.s }, p1/Z, [x21, x28, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z14.s }, p1/Z, [x21, x12, LSL #2]\n"
+ "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p1/Z, [x20, x12, LSL #2]\n"
+ "ld1w { z16.s }, p1/Z, [x19, x12, LSL #2]\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z10.s }, p1/Z, [x20, x28, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n"
+ "incw x12\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
- "ld1w { z11.s }, p1/Z, [x24, x28, LSL #2]\n"
- "prfm pldl1keep, [x24, x12]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x23, x28, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "ldp x22, x21, [x13, #0x20]\n"
- "ldp x20, x19, [x13, #0x30]\n"
- "st1w { z31.s }, p0, [x10, x25, LSL #2]\n"
- "ld1w { z13.s }, p1/Z, [x22, x28, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ld1w { z14.s }, p1/Z, [x21, x28, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z15.s }, p1/Z, [x20, x28, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
- "ld1w { z16.s }, p1/Z, [x19, x28, LSL #2]\n"
- "incw x28\n"
- "prfm pldl1keep, [x19, x12]\n"
- "cmp x28, %x[n_channels]\n"
- "st1w { z30.s }, p0, [x9, x25, LSL #2]\n"
- "st1w { z29.s }, p0, [x27, x25, LSL #2]\n"
- "st1w { z28.s }, p0, [x26, x25, LSL #2]\n"
- "ld1w { z17.s }, p3/Z, [x14]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
- "mov z30.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
- "addvl x14, x14, #16\n"
- "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
- "addvl x14, x14, #-6\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x13, #0x40]\n"
- "incw x25\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "ldr x21, [x13, #0x48]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x40]\n"
+ "incw x9\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "ldr x25, [x14, #0x48]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x20, [x13, #0x50]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ldr x19, [x13, #0x58]\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
+ "ldr x24, [x14, #0x50]\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.s, p3/M, z0.s, z16.s\n"
- "ldr x21, [x13, #0x60]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.s, p3/M, z3.s, z14.s\n"
- "ldr x20, [x13, #0x68]\n"
- "ldr x19, [x13, #0x70]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z15.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "ldr x19, [x13, #0x78]\n"
- "ldr x21, [x13, #0x80]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "ldr x20, [x13, #0x88]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x24, [x13, #0x90]\n"
+ "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z16.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.s, p3/M, z6.s, z15.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z30.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x24, x12]\n"
- "ldr x23, [x13, #0x98]\n"
"fmla z29.s, p3/M, z6.s, z15.s\n"
- "ldr x22, [x13, #0xa0]\n"
- "fmax z31.s, p3/M, z31.s, z19.s\n"
- "ldr x21, [x13, #0xa8]\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "fmin z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
+ "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z11.s\n"
- "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "fmax z31.s, p3/M, z31.s, z19.s\n"
"fmla z29.s, p3/M, z7.s, z13.s\n"
- "ldr x20, [x13, #0xb0]\n"
"fmax z30.s, p3/M, z30.s, z19.s\n"
- "ldr x19, [x13, #0xb8]\n"
- "ldr x22, [x13, #0xc0]\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "fmin z31.s, p3/M, z31.s, z18.s\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z16.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z15.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "st1w { z31.s }, p0, [x10, x25, LSL #2]\n"
"fmla z28.s, p3/M, z6.s, z15.s\n"
- "st1w { z30.s }, p0, [x9, x25, LSL #2]\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
- "st1w { z29.s }, p0, [x27, x25, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z28.s }, p0, [x26, x25, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 6f1f187818..751874ffbf 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@ namespace depthwise {
void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst
+class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@ struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index edafe82770..ea8bbbd7e8 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -114,22 +114,18 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x10, x20, x22, LSL #2\n"
"ld1w { z16.s }, p3/Z, [x8]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
"add x9, x10, x22, LSL #2\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
"add x28, x9, x22, LSL #2\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
"add x27, x28, x22, LSL #2\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
"add x26, x15, x15\n"
- "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
"add x25, x26, x15\n"
- "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
+ "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
"add x24, x25, x15\n"
+ "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
"add x23, x24, x15\n"
"madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col
"mul x19, x19, x7\n" // offset *= output_tile_size
@@ -150,14 +146,14 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
"ld1w { z14.s }, p2/Z, [x10]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z0.s, z5.s\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n"
"whilelt p1.s, x16, %x[n_channels]\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
"incw x14\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
"ld1w { z0.s }, p3/Z, [x8]\n"
"incw x17\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
@@ -334,29 +330,25 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
"fmax z28.s, p3/M, z28.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
"st1w { z31.s }, p0, [x11]\n"
- "mov z31.d, z16.d\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
+ "fmin z29.s, p3/M, z29.s, z17.s\n"
"st1w { z30.s }, p0, [x11, x12, LSL #2]\n"
- "mov z30.d, z16.d\n"
+ "fmin z28.s, p3/M, z28.s, z17.s\n"
"addvl x11, x11, #1\n"
- "fmin z29.s, p3/M, z29.s, z17.s\n"
"st1w { z29.s }, p0, [x22]\n"
- "mov z29.d, z16.d\n"
- "fmin z28.s, p3/M, z28.s, z17.s\n"
"st1w { z28.s }, p0, [x22, x12, LSL #2]\n"
- "mov z28.d, z16.d\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z0.s, z5.s\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
"ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"add x21, x5, #0x1\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
"ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
"ld1w { z0.s }, p3/Z, [x8]\n"
"add x6, x6, #0x1\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
@@ -528,4 +520,4 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 1bfe7eb09c..070270764c 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -100,511 +100,429 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x14, ALL, MUL #2\n"
- "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"mov x13, #0x0\n"
- "ldp x12, x11, [x19, #0x0]\n"
- "cntw x10\n"
- "ldp x9, x28, [x19, #0x10]\n"
- "sub x27, XZR, x10\n"
- "ld1w { z16.s }, p3/Z, [x16]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
+ "cntw x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
- "cmp x10, %x[n_channels]\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
- "addvl x16, x16, #6\n"
- "ldp x26, x25, [x15, #0x0]\n"
- "ldp x24, x23, [x15, #0x10]\n"
- "ldp x20, x19, [x15, #0x20]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #6\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
"ld1w { z5.s }, p2/Z, [x26, x13, LSL #2]\n"
- "prfm pldl1keep, [x26, x14]\n"
"ld1w { z6.s }, p2/Z, [x25, x13, LSL #2]\n"
- "prfm pldl1keep, [x25, x14]\n"
"ld1w { z7.s }, p2/Z, [x24, x13, LSL #2]\n"
- "prfm pldl1keep, [x24, x14]\n"
"ld1w { z8.s }, p2/Z, [x23, x13, LSL #2]\n"
- "prfm pldl1keep, [x23, x14]\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
- "prfm pldl1keep, [x19, x14]\n"
- "ldp x22, x21, [x15, #0x30]\n"
- "ldp x20, x19, [x15, #0x40]\n"
- "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n"
- "prfm pldl1keep, [x22, x14]\n"
- "ld1w { z12.s }, p2/Z, [x21, x13, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
- "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z0.s, z5.s\n"
- "ldr x21, [x15, #0x50]\n"
- "whilelt p1.s, x10, %x[n_channels]\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0x58]\n"
- "incw x27\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
- "ldr x20, [x15, #0x60]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
+ "ldr x24, [x14, #0x50]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
+ "ldr x23, [x14, #0x58]\n"
+ "incw x9\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
+ "ldr x22, [x14, #0x60]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z0.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.s, p3/M, z1.s, z8.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z28.s, p3/M, z1.s, z13.s\n"
- "ld1w { z0.s }, p3/Z, [x16]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0x68]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.s, p3/M, z2.s, z13.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z28.s, p3/M, z2.s, z5.s\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.s, p3/M, z3.s, z5.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z28.s, p3/M, z3.s, z6.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z9.s\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z6.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "ldr x26, [x15, #0x80]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z7.s\n"
- "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.s, p3/M, z0.s, z8.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.s, p3/M, z0.s, z14.s\n"
- "prfm pldl1keep, [x26, x14]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z8.s\n"
- "ldr x25, [x15, #0x88]\n"
+ "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z13.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.s, p3/M, z1.s, z11.s\n"
- "ldr x24, [x15, #0x90]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z5.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z28.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x15, #0x98]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.s, p3/M, z3.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z3.s, z6.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "ld1w { z16.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "ldr x20, [x15, #0xa0]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z6.s\n"
"ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "addvl x16, x16, #16\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z28.s, p3/M, z4.s, z8.s\n"
- "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z14.s\n"
- "ldr x19, [x15, #0xa8]\n"
+ "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z0.s, z11.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.s, p3/M, z0.s, z5.s\n"
- "ldr x22, [x15, #0xb0]\n"
"fmla z28.s, p3/M, z0.s, z6.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x22, x14]\n"
"fmla z29.s, p3/M, z1.s, z6.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n"
"fmla z28.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x15, #0xb8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z2.s, z9.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z10.s\n"
- "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "ldr x20, [x15, #0xc0]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x100]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x19, [x15, #0xc8]\n"
"fmla z29.s, p3/M, z3.s, z11.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x25, [x14, #0x108]\n"
"fmla z30.s, p3/M, z4.s, z8.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z28.s, p3/M, z4.s, z14.s\n"
- "ldr x21, [x15, #0xd0]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z5.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0x110]\n"
"fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0xd8]\n"
"fmla z29.s, p3/M, z0.s, z9.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x118]\n"
"fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z29.s, p3/M, z1.s, z13.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n"
"fmla z28.s, p3/M, z1.s, z5.s\n"
- "ldr x20, [x15, #0xe0]\n"
+ "ld1w { z1.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z2.s, z10.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0xe8]\n"
"fmla z29.s, p3/M, z2.s, z5.s\n"
- "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z6.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z29.s, p3/M, z3.s, z6.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n"
"fmla z28.s, p3/M, z3.s, z8.s\n"
- "ldr x20, [x15, #0xf0]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
- "ldr x19, [x15, #0xf8]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z14.s\n"
- "ld1w { z1.s }, p3/Z, [x16]\n"
"fmla z29.s, p3/M, z4.s, z8.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z9.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ldr x26, [x15, #0x100]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "ldr x25, [x15, #0x108]\n"
"fmla z30.s, p3/M, z1.s, z5.s\n"
- "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z9.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z5.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z5.s }, p1/Z, [x26, x12, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z6.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x24, [x15, #0x110]\n"
- "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "ldr x23, [x15, #0x118]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
"fmla z31.s, p3/M, z3.s, z6.s\n"
+ "ld1w { z6.s }, p1/Z, [x25, x12, LSL #2]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
"fmla z30.s, p3/M, z3.s, z8.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
"fmla z29.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z7.s }, p1/Z, [x24, x12, LSL #2]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "incw x13\n"
+ "ld1w { z13.s }, p1/Z, [x21, x12, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z8.s\n"
- "ldp x26, x25, [x15, #0x0]\n"
- "whilelt p2.s, x13, %x[n_channels]\n"
+ "ld1w { z8.s }, p1/Z, [x23, x12, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "ldp x24, x23, [x15, #0x10]\n"
- "addvl x14, x14, #1\n"
+ "ld1w { z11.s }, p1/Z, [x20, x12, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "ldp x20, x19, [x15, #0x20]\n"
- "ldp x22, x21, [x15, #0x30]\n"
+ "ld1w { z12.s }, p1/Z, [x19, x12, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z9.s\n"
- "ld1w { z5.s }, p1/Z, [x26, x10, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x22, x12, LSL #2]\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "prfm pldl1keep, [x26, x14]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z6.s }, p1/Z, [x25, x10, LSL #2]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "addvl x15, x15, #16\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z7.s }, p1/Z, [x24, x10, LSL #2]\n"
+ "ld1w { z10.s }, p1/Z, [x26, x12, LSL #2]\n"
+ "ld1w { z14.s }, p1/Z, [x25, x12, LSL #2]\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "prfm pldl1keep, [x24, x14]\n"
+ "incw x12\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "ld1w { z8.s }, p1/Z, [x23, x10, LSL #2]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "prfm pldl1keep, [x23, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "ld1w { z9.s }, p1/Z, [x20, x10, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z13.s }, p1/Z, [x19, x10, LSL #2]\n"
- "prfm pldl1keep, [x19, x14]\n"
- "ld1w { z11.s }, p1/Z, [x22, x10, LSL #2]\n"
- "prfm pldl1keep, [x22, x14]\n"
- "ld1w { z12.s }, p1/Z, [x21, x10, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
- "ldp x20, x19, [x15, #0x40]\n"
- "st1w { z31.s }, p0, [x12, x27, LSL #2]\n"
- "st1w { z30.s }, p0, [x11, x27, LSL #2]\n"
- "ld1w { z10.s }, p1/Z, [x20, x10, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z14.s }, p1/Z, [x19, x10, LSL #2]\n"
- "incw x10\n"
- "prfm pldl1keep, [x19, x14]\n"
- "cmp x10, %x[n_channels]\n"
- "st1w { z29.s }, p0, [x9, x27, LSL #2]\n"
- "st1w { z28.s }, p0, [x28, x27, LSL #2]\n"
- "ld1w { z16.s }, p3/Z, [x16, #4, MUL VL]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n"
- "addvl x16, x16, #16\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n"
- "addvl x16, x16, #-6\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z0.s, z5.s\n"
- "ldr x21, [x15, #0x50]\n"
- "incw x27\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0x58]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
+ "ldr x24, [x14, #0x50]\n"
+ "incw x9\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
+ "ldr x23, [x14, #0x58]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
- "ldr x20, [x15, #0x60]\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
+ "ldr x22, [x14, #0x60]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z0.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.s, p3/M, z1.s, z8.s\n"
"fmla z28.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z0.s }, p3/Z, [x16]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0x68]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.s, p3/M, z2.s, z13.s\n"
"fmla z28.s, p3/M, z2.s, z5.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.s, p3/M, z3.s, z5.s\n"
"fmla z28.s, p3/M, z3.s, z6.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z9.s\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z6.s\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n"
- "ldr x26, [x15, #0x80]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.s, p3/M, z0.s, z7.s\n"
- "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.s, p3/M, z0.s, z8.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.s, p3/M, z0.s, z14.s\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z31.s, p3/M, z1.s, z8.s\n"
- "ldr x25, [x15, #0x88]\n"
+ "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z13.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.s, p3/M, z1.s, z11.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z31.s, p3/M, z2.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z5.s\n"
- "ldr x24, [x15, #0x90]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
"fmla z28.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x15, #0x98]\n"
- "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.s, p3/M, z3.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z3.s, z6.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "ldr x20, [x15, #0xa0]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z6.s\n"
"ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "addvl x16, x16, #16\n"
"fmla z28.s, p3/M, z4.s, z8.s\n"
- "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z31.s, p3/M, z0.s, z14.s\n"
- "ldr x19, [x15, #0xa8]\n"
+ "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z0.s, z11.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.s, p3/M, z0.s, z5.s\n"
- "ldr x22, [x15, #0xb0]\n"
"fmla z28.s, p3/M, z0.s, z6.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x22, x14]\n"
"fmla z29.s, p3/M, z1.s, z6.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n"
"fmla z28.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x15, #0xb8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z2.s, z9.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z10.s\n"
- "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "ldr x20, [x15, #0xc0]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x100]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x19, [x15, #0xc8]\n"
"fmla z29.s, p3/M, z3.s, z11.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x25, [x14, #0x108]\n"
"fmla z30.s, p3/M, z4.s, z8.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z28.s, p3/M, z4.s, z14.s\n"
- "ldr x21, [x15, #0xd0]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z5.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0x110]\n"
"fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0xd8]\n"
"fmla z29.s, p3/M, z0.s, z9.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x118]\n"
"fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z29.s, p3/M, z1.s, z13.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n"
"fmla z28.s, p3/M, z1.s, z5.s\n"
- "ldr x20, [x15, #0xe0]\n"
+ "ld1w { z1.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z2.s, z10.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0xe8]\n"
"fmla z29.s, p3/M, z2.s, z5.s\n"
- "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z6.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z29.s, p3/M, z3.s, z6.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n"
"fmla z28.s, p3/M, z3.s, z8.s\n"
- "ldr x20, [x15, #0xf0]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
- "ldr x19, [x15, #0xf8]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z14.s\n"
- "ld1w { z1.s }, p3/Z, [x16]\n"
"fmla z29.s, p3/M, z4.s, z8.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z9.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ldr x26, [x15, #0x100]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "ldr x25, [x15, #0x108]\n"
"fmla z30.s, p3/M, z1.s, z5.s\n"
- "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z9.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z5.s\n"
- "prfm pldl1keep, [x25, x14]\n"
"fmla z30.s, p3/M, z2.s, z6.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x24, [x15, #0x110]\n"
- "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "ldr x23, [x15, #0x118]\n"
"fmla z31.s, p3/M, z3.s, z6.s\n"
"fmla z30.s, p3/M, z3.s, z8.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z29.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z31.s, p3/M, z4.s, z8.s\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
@@ -612,22 +530,22 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(
"fmax z31.s, p3/M, z31.s, z18.s\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x12, x27, LSL #2]\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x11, x27, LSL #2]\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z29.s }, p0, [x9, x27, LSL #2]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z28.s }, p0, [x28, x27, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (&params_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index c501c67a5b..b524fd7c93 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -29,7 +29,7 @@
#pragma once
-#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2)
+#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -72,4 +72,4 @@ struct sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2)
+#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
index 40220ad84e..52dc4683ad 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
index 39ab3534f5..34ba8ece12 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
index 7f4272672c..441da6da7a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>