diff options
author | Georgios Pinitas <georgios.pinitas@arm.com> | 2019-10-14 19:03:09 +0100 |
---|---|---|
committer | Georgios Pinitas <georgios.pinitas@arm.com> | 2019-10-23 12:08:12 +0000 |
commit | 48b3ef89de5f21a0169d8416e3d54081f82c7bf8 (patch) | |
tree | f857d733ccf446c704823dc7ac796a96eb55095e /src/core/NEON/kernels | |
parent | 1dce3101ef8d77c8cf0af7dfd4af6595a0136b91 (diff) | |
download | ComputeLibrary-48b3ef89de5f21a0169d8416e3d54081f82c7bf8.tar.gz |
COMPMID-2577: Fuse bias addition and activation in gemm assembly kernels
Change-Id: I7f52112d2d05b1ea3d3f3d4b19b8eafab05d6c44
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/2141
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez <pablo.tello@arm.com>
Diffstat (limited to 'src/core/NEON/kernels')
106 files changed, 45943 insertions, 40238 deletions
diff --git a/src/core/NEON/kernels/NEActivationLayerKernel.cpp b/src/core/NEON/kernels/NEActivationLayerKernel.cpp index 6f722e0457..c338ef09c7 100644 --- a/src/core/NEON/kernels/NEActivationLayerKernel.cpp +++ b/src/core/NEON/kernels/NEActivationLayerKernel.cpp @@ -109,7 +109,7 @@ std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITen } // namespace NEActivationLayerKernel::NEActivationLayerKernel() - : _input(nullptr), _output(nullptr), _func(nullptr), _act_info(ActivationFunction::LOGISTIC) + : _input(nullptr), _output(nullptr), _func(nullptr), _act_info() { } @@ -121,11 +121,18 @@ void NEActivationLayerKernel::configure(ITensor *input, ITensor *output, Activat _act_info = activation_info; _output = input; + // Out-of-place calculation if(output != nullptr) { _output = output; } + // Disabled activation, thus no operation needed + if(!activation_info.enabled()) + { + _func = nullptr; + } + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), (output != nullptr) ? output->info() : nullptr, activation_info)); // Activation functions : FP32 @@ -610,6 +617,12 @@ Status NEActivationLayerKernel::validate(const ITensorInfo *input, const ITensor void NEActivationLayerKernel::run(const Window &window, const ThreadInfo &info) { + // Early exit on disabled activation + if(!_act_info.enabled()) + { + return; + } + ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window); diff --git a/src/core/NEON/kernels/arm_gemm/bias_adder.hpp b/src/core/NEON/kernels/arm_gemm/bias_adder.hpp new file mode 100644 index 0000000000..745d00563b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/bias_adder.hpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +namespace arm_gemm +{ + +// Fallback routine to add bias to a block +template<typename T> +inline void bias_adder(T *out, unsigned int stride, const T *bias, unsigned int rows, unsigned int cols) { + for (unsigned int row=0; row<rows; row++) { + for (unsigned int col=0; col<cols; col++) { + out[row * stride + col] += bias[col]; + } + } +} + +template<bool DoBias, typename T> +inline void activator(T *out, unsigned int stride, const T *bias, Activation act, unsigned int rows, unsigned int cols) { + if (act.type == Activation::Type::None) { + if (DoBias) { + bias_adder(out, stride, bias, rows, cols); + } + return; + } + + if (act.type == Activation::Type::ReLU) { + for (unsigned int row=0; row<rows; row++) { + for (unsigned int col=0; col<cols; col++) { + T &v = out[row * stride + col]; + if (DoBias) { + v += bias[col]; + } + v = std::max(static_cast<T>(0), v); + } + } + } + + if (act.type == Activation::Type::BoundedReLU) { + const T max = static_cast<T>(act.param1); + + for (unsigned int row=0; row<rows; row++) { + for (unsigned int col=0; col<cols; col++) { + T &v = out[row * stride + col]; + if (DoBias) { + v += bias[col]; + } + v = std::max(static_cast<T>(0), std::min(v, max)); + } + } + } +} + +} // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp index 8541d34de5..b6671e8c85 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp @@ -47,23 +47,23 @@ static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = { { GemmMethod::GEMM_HYBRID, "hybrid_fp16_mla_4VLx4", - [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<__fp16> &args) { return new GemmHybrid<hybrid_fp16_mla_4VLx4, __fp16, __fp16>(args); } + [](const GemmArgs &args) { return (args._Ksize >= 8) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp16_mla_4VLx4, __fp16, __fp16>(args); } }, { GemmMethod::GEMM_NATIVE, "native_fp16_mla_4VLx4", - [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8 && args._alpha==1.0f && !args._trA && !args._trB); }, - [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<__fp16> &args) { return new GemmNative<native_fp16_mla_4VLx4, __fp16, __fp16>(args); } + [](const GemmArgs &args) { return (args._Ksize >= 8 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative<native_fp16_mla_4VLx4, __fp16, __fp16>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "interleaved_fp16_mla_3VLx8", - [](const GemmArgs<__fp16> &args) { return (args._Ksize > 4); }, + [](const GemmArgs &args) { return (args._Ksize > 4); }, nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); } }, #endif @@ -72,21 +72,21 @@ static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = { GemmMethod::GEMM_INTERLEAVED, "hgemm_24x8", #ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - [](const GemmArgs<__fp16> &args) { return args._ci->has_fp16(); }, + [](const GemmArgs &args) { return args._ci->has_fp16(); }, #else nullptr, #endif nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); } }, -#endif +#endif // aarch64 && FP16 #ifdef __aarch64__ { GemmMethod::GEMM_INTERLEAVED, "sgemm_12x8", nullptr, nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args); } }, #elif defined(__arm__) { @@ -94,7 +94,7 @@ static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = { "sgemm_8x6", nullptr, nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved<sgemm_8x6, __fp16, __fp16>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<sgemm_8x6, __fp16, __fp16>(args); } }, #else // not AArch64 or AArch32 # error Unknown Architecture @@ -114,9 +114,9 @@ const GemmImplementation<__fp16, __fp16> *gemm_implementation_list<__fp16, __fp1 } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &); -template KernelDescription get_gemm_method<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &); -template std::vector<KernelDescription> get_compatible_kernels<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &); +template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector<KernelDescription> get_compatible_kernels<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp index dedcdb7655..cf91ee0652 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp @@ -33,17 +33,16 @@ #include "kernels/a32_sgemm_8x6.hpp" #include "kernels/a64_hybrid_fp32_mla_16x4.hpp" +#include "kernels/a64_native_fp32_mla_16x4.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp" #include "kernels/a64_sgemm_12x8.hpp" -#include "kernels/a64_sgemm_native_16x4.hpp" -#include "kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp" #include "kernels/a64_sgemv_pretransposed.hpp" #include "kernels/a64_sgemv_trans.hpp" #include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp" #include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp" #include "kernels/sve_native_fp32_mla_4VLx4.hpp" -#include "kernels/sve_smallK_fp32_mla_1VLx4.hpp" -#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp" namespace arm_gemm { @@ -52,88 +51,81 @@ static const GemmImplementation<float, float> gemm_fp32_methods[] = { GemmMethod::GEMV_BATCHED, "gemv_batched", - [](const GemmArgs<float> &args) { return (args._Msize==1) && (args._nbatches>1); }, + [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); }, nullptr, - [](const GemmArgs<float> &args) { return new GemvBatched<float, float>(args); } + [](const GemmArgs &args) { return new GemvBatched<float, float>(args); } }, #ifdef __aarch64__ { GemmMethod::GEMV_PRETRANSPOSED, "sgemv_pretransposed", - [](const GemmArgs<float> &args) { return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1); }, + [](const GemmArgs &args) { return (args._Msize==1 && args._pretransposed_hint && args._nbatches==1); }, nullptr, - [](const GemmArgs<float> &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); } + [](const GemmArgs &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); } }, { GemmMethod::GEMV_NATIVE_TRANSPOSED, "sgemv_trans", - [](const GemmArgs<float> &args) { return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1); }, + [](const GemmArgs &args) { return (args._Msize==1 && !args._trA && !args._trB && args._nbatches==1); }, nullptr, - [](const GemmArgs<float> &args) { return new GemvNativeTransposed<sgemv_trans, float, float>(args); } + [](const GemmArgs &args) { return new GemvNativeTransposed<sgemv_trans, float, float>(args); } }, #ifdef __ARM_FEATURE_SVE -// SVE smallk / native / hybrid methods -{ - GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_1VLx4", - [](const GemmArgs<float> &args) { return (args._Ksize <= 24) && !args._trA && args._alpha==1.0f && args._pretransposed_hint; }, - nullptr, - [](const GemmArgs<float> &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_1VLx4, float, float>(args); } -}, +// SVE native / hybrid methods { GemmMethod::GEMM_HYBRID, "hybrid_fp32_mla_4VLx4", - [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<float> &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); } -}, -{ - GemmMethod::GEMM_NATIVE, - "smallK_fp32_mla_1VLx4", - [](const GemmArgs<float> &args) { return (args._Ksize <= 24) && !args._trA && !args._trB && args._alpha==1.0f; }, - nullptr, - [](const GemmArgs<float> &args) { return new GemmNative<smallK_fp32_mla_1VLx4, float, float>(args); } + [](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); } }, { GemmMethod::GEMM_NATIVE, "native_fp32_mla_4VLx4", - [](const GemmArgs<float> &args) { return (args._Ksize>4 && args._alpha==1.0f && !args._trA && !args._trB); }, - [](const GemmArgs<float> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<float> &args) { return new GemmNative<native_fp32_mla_4VLx4, float, float>(args); } + [](const GemmArgs &args) { return (args._Ksize>4 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative<native_fp32_mla_4VLx4, float, float>(args); } }, #endif // __ARM_FEATURE_SVE // NEON native / hybrid methods { GemmMethod::GEMM_HYBRID, - "sgemm_nativeA_pretransposeB_16x4", - [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<float> &args) { return new GemmHybrid<sgemm_nativeA_pretransposeB_16x4, float, float>(args); } + "smallK_hybrid_fp32_mla_4x8", + [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0 && !args._trA && args._pretransposed_hint; }, + nullptr, + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x8, float, float>(args); } +}, +{ + GemmMethod::GEMM_HYBRID, + "smallK_hybrid_fp32_mla_4x6", + [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._trA && args._pretransposed_hint; }, + nullptr, + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x6, float, float>(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_fp32_mla_16x4", - [](const GemmArgs<float> &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<float> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<float> &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); } + [](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); } }, { GemmMethod::GEMM_NATIVE, - "sgemm_native_16x4", - [](const GemmArgs<float> &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB); }, - [](const GemmArgs<float> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<float> &args) { return new GemmNative<sgemm_native_16x4, float, float>(args); } + "native_fp32_mla_16x4", + [](const GemmArgs &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative<native_fp32_mla_16x4, float, float>(args); } }, #ifdef __ARM_FEATURE_SVE { GemmMethod::GEMM_INTERLEAVED, "interleaved_fp32_mla_3VLx8", - [](const GemmArgs<float> &args) { return (args._Ksize>4); }, + [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs<float> &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); } }, #endif // __ARM_FEATURE_SVE { @@ -141,7 +133,7 @@ static const GemmImplementation<float, float> gemm_fp32_methods[] = "sgemm_12x8", nullptr, nullptr, - [](const GemmArgs<float> &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); } }, #endif // __aarch64__ @@ -151,7 +143,7 @@ static const GemmImplementation<float, float> gemm_fp32_methods[] = "sgemm_8x6", nullptr, nullptr, - [](const GemmArgs<float> &args) { return new GemmInterleaved<sgemm_8x6, float, float>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<sgemm_8x6, float, float>(args); } }, #endif // __arm__ { @@ -170,8 +162,8 @@ const GemmImplementation<float, float> *gemm_implementation_list<float, float>() } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<float, float> gemm<float, float, Nothing>(const GemmArgs<float> &args, const Nothing &); -template KernelDescription get_gemm_method<float, float, Nothing>(const GemmArgs<float> &args, const Nothing &); -template std::vector<KernelDescription> get_compatible_kernels<float, float, Nothing> (const GemmArgs<float> &args, const Nothing &); +template UniqueGemmCommon<float, float> gemm<float, float, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<float, float, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector<KernelDescription> get_compatible_kernels<float, float, Nothing> (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp index 436f55dee2..c3abb04db7 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2017-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include <algorithm> #include "arm_gemm.hpp" +#include "bias_adder.hpp" #include "ndrange.hpp" #include "utils.hpp" @@ -58,7 +59,7 @@ class GemmHybrid : public GemmCommon<To, Tr> { const bool _trB; - const Tr _beta; + const Activation _act; /* Blocking info */ const unsigned int _k_block; @@ -70,7 +71,12 @@ class GemmHybrid : public GemmCommon<To, Tr> { const NDRange<4> _window_range; - static unsigned int compute_k_block(const GemmArgs<Tr> &args) { + static unsigned int compute_k_block(const GemmArgs &args) { + // Some kernels don't support append mode - these can't do K blocking at all. + if (!strategy::supports_append()) { + return args._Ksize; + } + if (args._cfg && args._cfg->inner_block_size) { return args._cfg->inner_block_size; } @@ -97,7 +103,7 @@ class GemmHybrid : public GemmCommon<To, Tr> { return k_block; } - static unsigned int compute_n_block(const GemmArgs<Tr> &args) { + static unsigned int compute_n_block(const GemmArgs &args) { if (args._cfg && args._cfg->outer_block_size) { return args._cfg->outer_block_size; } @@ -127,9 +133,10 @@ public: GemmHybrid & operator= (GemmHybrid &) = delete; /* Constructor */ - GemmHybrid(const GemmArgs<Tr> &args) + GemmHybrid(const GemmArgs &args) : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), - _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta), + _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), + _act(args._act), _k_block(compute_k_block(args)), _n_block(compute_n_block(args)), _Mround(roundup(args._Msize, strategy::out_height())), _window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti) { } @@ -164,6 +171,9 @@ public: unsigned int kmax = std::min(k0 + _k_block, _Ksize); unsigned int kern_k = roundup(kmax-k0, strategy::k_unroll()); + const bool first_pass = (k0 == 0); + const bool last_pass = (kmax == _Ksize); + auto p = _window_range.iterator(start, end); if (p.done()) { @@ -190,8 +200,17 @@ public: strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda, b_panel, this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc, - (k0 == 0) ? _beta : static_cast<Tr>(1), - (m_end - m_start), (nmax - n0), kmax-k0); + (m_end - m_start), (nmax - n0), kmax-k0, + (strategy::supports_bias() && first_pass && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr, + last_pass ? _act : Activation(), !first_pass); + + // Add bias externally if needed + if (!strategy::supports_bias() && this->_bias && first_pass) { + bias_adder(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc, + this->_bias + (multi * this->_bias_multi_stride) + n0, + (m_end - m_start), (nmax - n0)); + } + } while (p.next_dim1()); } } diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp index b4edece8d5..574ecef5b2 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp @@ -58,8 +58,6 @@ class GemmHybridQuantized : public GemmCommon<To, Tr> { const bool _trB; - const Tr _beta; - /* Blocking info */ const unsigned int _k_block; const unsigned int _n_block; @@ -82,7 +80,7 @@ class GemmHybridQuantized : public GemmCommon<To, Tr> { return _Nsize * _nmulti * sizeof(int32_t); } - static unsigned int compute_k_block(const GemmArgs<Tr> &args) { + static unsigned int compute_k_block(const GemmArgs &args) { // We don't support K blocks as we only temporarily store 32 bit results. return args._Ksize; @@ -112,7 +110,7 @@ class GemmHybridQuantized : public GemmCommon<To, Tr> { return k_block; } - static unsigned int compute_n_block(const GemmArgs<Tr> &args) { + static unsigned int compute_n_block(const GemmArgs &args) { if (args._cfg && args._cfg->outer_block_size) { return args._cfg->outer_block_size; } @@ -142,9 +140,9 @@ public: GemmHybridQuantized & operator= (GemmHybridQuantized &) = delete; /* Constructor */ - GemmHybridQuantized(const GemmArgs<Tr> &args, const ARequantizeLayer32 &qp) + GemmHybridQuantized(const GemmArgs &args, const ARequantizeLayer32 &qp) : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), - _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta), + _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _k_block(compute_k_block(args)), _n_block(compute_n_block(args)), _Mround(roundup(args._Msize, strategy::out_height())), _window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti), @@ -210,8 +208,8 @@ public: strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda, b_panel, result_buffer, (nmax-n0), - (k0 == 0) ? _beta : static_cast<Tr>(1), - (m_end - m_start), (nmax - n0), kern_k); + (m_end - m_start), (nmax - n0), kern_k, + nullptr, Activation(), false); } { @@ -262,7 +260,7 @@ public: col_bias = reinterpret_cast<int32_t *>(in_buffer); for (unsigned int i=0; i<_nmulti; i++) { - compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, 0); + compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, i, 0); } uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer); @@ -295,8 +293,9 @@ public: col_bias = reinterpret_cast<int32_t *>(in_buffer); } - void set_quantized_bias(const int32_t *bias) override { + void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override { _qp.bias = bias; + _qp.bias_multi_stride = bias_multi_stride; } }; diff --git a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp index 55d72f88cb..569d1f44ca 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp @@ -34,13 +34,13 @@ namespace arm_gemm { */ template<typename Top, typename Tret, class OutputStage = Nothing> struct GemmImplementation { - const GemmMethod method; - const char * name; - std::function<bool(const GemmArgs<Tret> &, const OutputStage &)> is_supported; - std::function<bool(const GemmArgs<Tret> &, const OutputStage &)> is_recommended; - std::function<GemmCommon<Top, Tret> *(const GemmArgs<Tret> &, const OutputStage &)> instantiate; + const GemmMethod method; + const char * name; + std::function<bool(const GemmArgs &, const OutputStage &)> is_supported; + std::function<bool(const GemmArgs &, const OutputStage &)> is_recommended; + std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate; - bool do_is_supported(const GemmArgs<Tret> &args, const OutputStage &os) const { + bool do_is_supported(const GemmArgs &args, const OutputStage &os) const { if (is_supported != nullptr) { return is_supported(args, os); } else { @@ -48,7 +48,7 @@ struct GemmImplementation { } } - bool do_is_recommended(const GemmArgs<Tret> &args, const OutputStage &os) const { + bool do_is_recommended(const GemmArgs &args, const OutputStage &os) const { if (is_recommended != nullptr) { return is_recommended(args, os); } else { @@ -56,7 +56,7 @@ struct GemmImplementation { } } - GemmCommon<Top, Tret> *do_instantiate(const GemmArgs<Tret> &args, const OutputStage &os) const { + GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const OutputStage &os) const { return instantiate(args, os); } }; @@ -66,13 +66,13 @@ struct GemmImplementation { * unnecessary second argument. */ template<typename Top, typename Tret> struct GemmImplementation<Top, Tret, Nothing> { - const GemmMethod method; - const char * name; - std::function<bool(const GemmArgs<Tret> &)> is_supported; - std::function<bool(const GemmArgs<Tret> &)> is_recommended; - std::function<GemmCommon<Top, Tret> *(const GemmArgs<Tret> &)> instantiate; + const GemmMethod method; + const char * name; + std::function<bool(const GemmArgs &)> is_supported; + std::function<bool(const GemmArgs &)> is_recommended; + std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate; - bool do_is_supported(const GemmArgs<Tret> &args, const Nothing &) const { + bool do_is_supported(const GemmArgs &args, const Nothing &) const { if (is_supported != nullptr) { return is_supported(args); } else { @@ -80,7 +80,7 @@ struct GemmImplementation<Top, Tret, Nothing> { } } - bool do_is_recommended(const GemmArgs<Tret> &args, const Nothing &) const { + bool do_is_recommended(const GemmArgs &args, const Nothing &) const { if (is_recommended != nullptr) { return is_recommended(args); } else { @@ -88,7 +88,7 @@ struct GemmImplementation<Top, Tret, Nothing> { } } - GemmCommon<Top, Tret> *do_instantiate(const GemmArgs<Tret> &args, const Nothing &) const { + GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const Nothing &) const { return instantiate(args); } }; @@ -116,7 +116,7 @@ const GemmImplementation<Top, Tret, OutputStage> *gemm_implementation_list(); * reference. */ template<typename Top, typename Tret, class OutputStage> -bool find_implementation(const GemmArgs<Tret> &args, const OutputStage &os, const GemmImplementation<Top, Tret, OutputStage> * &impl) { +bool find_implementation(const GemmArgs &args, const OutputStage &os, const GemmImplementation<Top, Tret, OutputStage> * &impl) { auto gemms = gemm_implementation_list<Top, Tret, OutputStage>(); const GemmConfig *cfg = args._cfg; @@ -168,7 +168,7 @@ bool find_implementation(const GemmArgs<Tret> &args, const OutputStage &os, cons } template<typename Top, typename Tret, class OutputStage> -std::vector<KernelDescription> get_compatible_kernels(const GemmArgs<Tret> &args, const OutputStage &os) { +std::vector<KernelDescription> get_compatible_kernels(const GemmArgs &args, const OutputStage &os) { std::vector<KernelDescription> res; /* Find out what the default implementation in so we can set the flag accordingly later. */ @@ -190,7 +190,7 @@ std::vector<KernelDescription> get_compatible_kernels(const GemmArgs<Tret> &args } template<typename Top, typename Tret, class OutputStage> -UniqueGemmCommon<Top, Tret> gemm(const GemmArgs<Tret> &args, const OutputStage &os) { +UniqueGemmCommon<Top, Tret> gemm(const GemmArgs &args, const OutputStage &os) { const GemmImplementation<Top, Tret, OutputStage> *impl; if (find_implementation<Top, Tret, OutputStage>(args, os, impl)) { @@ -201,7 +201,7 @@ UniqueGemmCommon<Top, Tret> gemm(const GemmArgs<Tret> &args, const OutputStage & } template<typename Top, typename Tret, class OutputStage> -KernelDescription get_gemm_method(const GemmArgs<Tret> &args, const OutputStage &os) { +KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage &os) { const GemmImplementation<Top, Tret, OutputStage> *impl; if (find_implementation<Top, Tret>(args, os, impl)) { diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp index 8062c3092a..e3b4416f68 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp @@ -38,7 +38,7 @@ static const GemmImplementation<int16_t, int32_t> gemm_s16_methods[] = { "gemm_s16_12x8", nullptr, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args); } }, { GemmMethod::DEFAULT, @@ -55,9 +55,9 @@ const GemmImplementation<int16_t, int32_t> *gemm_implementation_list<int16_t, in } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<int16_t, int32_t> gemm<int16_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &); -template KernelDescription get_gemm_method<int16_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &); -template std::vector<KernelDescription> get_compatible_kernels<int16_t, int32_t, Nothing> (const GemmArgs<int32_t> &args, const Nothing &); +template UniqueGemmCommon<int16_t, int32_t> gemm<int16_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<int16_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector<KernelDescription> get_compatible_kernels<int16_t, int32_t, Nothing> (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp index 49241a7129..a3446b9ddc 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp @@ -48,66 +48,66 @@ static const GemmImplementation<int8_t, int32_t> gemm_s8_methods[] = { { GemmMethod::GEMM_HYBRID, "smallK_hybrid_s8s32_dot_1VLx8", - [](const GemmArgs<int32_t> &args) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_1VLx8, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_1VLx8, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_s8s32_dot_4VLx4", - [](const GemmArgs<int32_t> &args) { return args._Ksize>=16 && args._alpha==1 && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<int32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<int32_t> &args) { return new GemmHybrid<hybrid_s8s32_dot_4VLx4, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return args._Ksize>=16 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_s8s32_dot_4VLx4, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_NATIVE, "native_s8s32_dot_4VLx4", - [](const GemmArgs<int32_t> &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); }, - [](const GemmArgs<int32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, - [](const GemmArgs<int32_t> &args) { return new GemmNative<native_s8s32_dot_4VLx4, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return (args._Ksize>=16 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, + [](const GemmArgs &args) { return new GemmNative<native_s8s32_dot_4VLx4, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "interleaved_s8s32_dot_3VLx8", - [](const GemmArgs<int32_t> &args) { return (args._Ksize>4) && args._alpha==1 && (args._beta == 0 || args._beta==1); }, + [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); } }, #endif { GemmMethod::GEMM_HYBRID, "smallK_hybrid_s8s32_dot_4x8", - [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x8, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x8, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_HYBRID, "smallK_hybrid_s8s32_dot_4x6", - [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x6, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x6, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_s8s32_dot_16x4", - [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs<int32_t> &args) { return args._Nsize<=256 && args._Ksize>128; }, - [](const GemmArgs<int32_t> &args) { return new GemmHybrid<hybrid_s8s32_dot_16x4, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_s8s32_dot_16x4, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_s8_12x8", - [](const GemmArgs<int32_t> &args) { return args._ci->has_dotprod() && args._alpha==1 && (args._beta==0 || args._beta==1); }, + [](const GemmArgs &args) { return args._ci->has_dotprod(); }, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_s8_4x4", nullptr, nullptr, - [](const GemmArgs<int32_t> &args) { return new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args); } }, { GemmMethod::DEFAULT, @@ -124,9 +124,9 @@ const GemmImplementation<int8_t, int32_t> *gemm_implementation_list<int8_t, int3 } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<int8_t, int32_t> gemm<int8_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &); -template KernelDescription get_gemm_method<int8_t, int32_t, Nothing>(const GemmArgs<int32_t> &args, const Nothing &); -template std::vector<KernelDescription> get_compatible_kernels<int8_t, int32_t, Nothing> (const GemmArgs<int32_t> &args, const Nothing &); +template UniqueGemmCommon<int8_t, int32_t> gemm<int8_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<int8_t, int32_t, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector<KernelDescription> get_compatible_kernels<int8_t, int32_t, Nothing> (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp index faff9acd2e..efd984561d 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp @@ -68,8 +68,7 @@ class GemmInterleaved : public GemmCommon<To, Tr> { const bool _trA; const bool _trB; - const Tr _alpha; - const Tr _beta; + const Activation _act; const int _maxthreads; int _nthreads; @@ -297,9 +296,14 @@ class GemmInterleaved : public GemmCommon<To, Tr> { #ifdef CYCLE_PROFILING auto p=prof.ScopedProfiler(PROFILE_MERGE, (strategy::out_height() * bblocks * strategy::out_width() * sizeof(Tr))); #endif + /* Only activate on last pass, only add bias on first pass, ask for accumulation on any non-first pass */ + const bool first_pass = current.k0()==0; + const bool last_pass = current.kmax()==_Ksize; + strat.transforms.Merge(this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride), c_panel, this->_ldc, y, ymax, current.x0(), current.xmax(), - _alpha, (current.k0()==0 ? _beta : static_cast<Tr>(1))); + ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr), + (last_pass ? _act : Activation()), !first_pass); } } } @@ -317,10 +321,10 @@ public: GemmInterleaved & operator= (GemmInterleaved &) = delete; /* Constructor */ - GemmInterleaved(const GemmArgs<Tr> &args) + GemmInterleaved(const GemmArgs &args) : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), _nbatches(args._nbatches), _nmulti(args._nmulti), _trA(args._trA), _trB(args._trB), - _alpha(args._alpha), _beta(args._beta), _maxthreads(args._maxthreads), _nthreads(args._maxthreads), + _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads), _pretransposed(args._pretransposed_hint) { const unsigned int L1_size = _ci->get_L1_cache_size(); const unsigned int L2_size = _ci->get_L2_cache_size(); diff --git a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp index ba9163b29b..fe6ebef045 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp @@ -54,7 +54,7 @@ class GemmNative : public GemmCommon<To, Tr> { const unsigned int _nbatches; const unsigned int _nmultis; - const Tr _beta; + const Activation _act; const CPUInfo * const _ci; @@ -63,11 +63,11 @@ class GemmNative : public GemmCommon<To, Tr> { const NDRange<4> _window_range; - static unsigned int compute_k_block(const GemmArgs<Tr> &args) { + static unsigned int compute_k_block(const GemmArgs &args) { return args._Ksize; } - static unsigned int compute_n_block(const GemmArgs<Tr> &args) { + static unsigned int compute_n_block(const GemmArgs &args) { if ((args._cfg != nullptr) && args._cfg->outer_block_size > 0) { return args._cfg->outer_block_size; } else { @@ -79,10 +79,10 @@ public: GemmNative(GemmNative &) = delete; GemmNative & operator= (GemmNative &) = delete; - GemmNative(const GemmArgs<Tr> &args) + GemmNative(const GemmArgs &args) : _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), _nbatches(args._nbatches), _nmultis(args._nmulti), - _beta(args._beta), _ci(args._ci), + _act(args._act), _ci(args._ci), _k_block(compute_k_block(args)), _n_block(compute_n_block(args)), _window_range(iceildiv(_Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmultis) { } @@ -127,7 +127,16 @@ public: strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (y0 * this->_lda), this->_lda, this->_Bptr + (multi * this->_B_multi_stride) + n0, this->_ldb, this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (y0 * this->_ldc) + n0, this->_ldc, - _beta, (ymax-y0), (nmax - n0), _Ksize); + (ymax-y0), (nmax-n0), _Ksize, + (strategy::supports_bias() && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr, + _act, false); + + // Add bias externally if needed + if (!strategy::supports_bias() && this->_bias) { + bias_adder(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (y0 * this->_ldc) + n0, this->_ldc, + this->_bias + (multi * this->_bias_multi_stride) + n0, + (ymax - y0), (nmax - n0)); + } } while (p.next_dim1()); } }; diff --git a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp index a402332552..079c04ae06 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp @@ -42,45 +42,45 @@ static const GemmImplementation<uint8_t, uint8_t, ARequantizeLayer32> gemm_quint { GemmMethod::GEMM_HYBRID_QUANTIZED, "smallK_hybrid_u8u32_dot_1VLx8", - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint8_t>(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint8_t>(args, qp); } }, { GemmMethod::GEMM_HYBRID_QUANTIZED, "hybrid_u8u32_dot_4VLx4", - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_4VLx4, uint8_t, uint8_t>(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_4VLx4, uint8_t, uint8_t>(args, qp); } }, #endif { GemmMethod::GEMM_HYBRID_QUANTIZED, "smallK_hybrid_u8u32_dot_4x8", - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint8_t>(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint8_t>(args, qp); } }, { GemmMethod::GEMM_HYBRID_QUANTIZED, "smallK_hybrid_u8u32_dot_4x6", - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint8_t>(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint8_t>(args, qp); } }, { GemmMethod::GEMM_HYBRID_QUANTIZED, "hybrid_u8u32_dot_16x4", - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &) { return args._Nsize<=256 && args._Ksize>128; }, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_16x4, uint8_t, uint8_t>(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Nsize<=256 && args._Ksize>128; }, + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_16x4, uint8_t, uint8_t>(args, qp); } }, { GemmMethod::QUANTIZE_WRAPPER, "quantized_wrapper", nullptr, nullptr, - [](const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &qp) { return new QuantizeWrapper<uint8_t, uint8_t, uint32_t>(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new QuantizeWrapper<uint8_t, uint8_t, uint32_t>(args, qp); } }, { GemmMethod::DEFAULT, @@ -96,9 +96,9 @@ const GemmImplementation<uint8_t, uint8_t, ARequantizeLayer32> *gemm_implementat return gemm_quint8_methods; } -template UniqueGemmCommon<uint8_t, uint8_t> gemm<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &os); -template KernelDescription get_gemm_method<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &os); -template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs<uint8_t> &args, const ARequantizeLayer32 &os); +template UniqueGemmCommon<uint8_t, uint8_t> gemm<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs &args, const ARequantizeLayer32 &os); +template KernelDescription get_gemm_method<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs &args, const ARequantizeLayer32 &os); +template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint8_t, ARequantizeLayer32>(const GemmArgs &args, const ARequantizeLayer32 &os); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp index 408911b6d5..85a8a6720a 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp @@ -38,7 +38,7 @@ static const GemmImplementation<uint16_t, uint32_t> gemm_u16_methods[] = { "gemm_u16_12x8", nullptr, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args); } }, { GemmMethod::DEFAULT, @@ -55,9 +55,9 @@ const GemmImplementation<uint16_t, uint32_t> *gemm_implementation_list<uint16_t, } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<uint16_t, uint32_t> gemm<uint16_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &); -template KernelDescription get_gemm_method<uint16_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &); -template std::vector<KernelDescription> get_compatible_kernels<uint16_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &); +template UniqueGemmCommon<uint16_t, uint32_t> gemm<uint16_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<uint16_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector<KernelDescription> get_compatible_kernels<uint16_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp index f5a30720d2..aead814d7e 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp @@ -48,66 +48,66 @@ static const GemmImplementation<uint8_t, uint32_t> gemm_u8_methods[] = { { GemmMethod::GEMM_HYBRID, "smallK_hybrid_u8u32_dot_1VLx8", - [](const GemmArgs<uint32_t> &args) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_u8u32_dot_4VLx4", - [](const GemmArgs<uint32_t> &args) { return args._Ksize>=16 && args._alpha==1 && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<uint32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<hybrid_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return args._Ksize>=16 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_NATIVE, "native_u8u32_dot_4VLx4", - [](const GemmArgs<uint32_t> &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); }, - [](const GemmArgs<uint32_t> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, - [](const GemmArgs<uint32_t> &args) { return new GemmNative<native_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return (args._Ksize>=16 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, + [](const GemmArgs &args) { return new GemmNative<native_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "interleaved_u8u32_dot_3VLx8", - [](const GemmArgs<uint32_t> &args) { return (args._Ksize>4) && args._alpha==1 && (args._beta==0 || args._beta==1); }, + [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); } }, #endif { GemmMethod::GEMM_HYBRID, "smallK_hybrid_u8u32_dot_4x8", - [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_HYBRID, "smallK_hybrid_u8u32_dot_4x6", - [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_u8u32_dot_16x4", - [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs<uint32_t> &args) { return args._Nsize<=256 && args._Ksize>128; }, - [](const GemmArgs<uint32_t> &args) { return new GemmHybrid<hybrid_u8u32_dot_16x4, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; }, + [](const GemmArgs &args) { return new GemmHybrid<hybrid_u8u32_dot_16x4, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_u8_12x8", - [](const GemmArgs<uint32_t> &args) { return args._ci->has_dotprod() && args._alpha==1 && (args._beta==0 || args._beta==1); }, + [](const GemmArgs &args) { return args._ci->has_dotprod(); }, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_u8_4x4", nullptr, nullptr, - [](const GemmArgs<uint32_t> &args) { return new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args); } }, { GemmMethod::DEFAULT, @@ -124,9 +124,9 @@ const GemmImplementation<uint8_t, uint32_t> *gemm_implementation_list<uint8_t, u } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<uint8_t, uint32_t> gemm<uint8_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &); -template KernelDescription get_gemm_method<uint8_t, uint32_t, Nothing>(const GemmArgs<uint32_t> &args, const Nothing &); -template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint32_t, Nothing> (const GemmArgs<uint32_t> &args, const Nothing &); +template UniqueGemmCommon<uint8_t, uint32_t> gemm<uint8_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<uint8_t, uint32_t, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector<KernelDescription> get_compatible_kernels<uint8_t, uint32_t, Nothing> (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp index 4453ee8243..be2f5614be 100644 --- a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp @@ -36,9 +36,9 @@ private: UniqueGemmCommon<To, Tr> _subgemm = nullptr; public: - GemvBatched(const GemmArgs<Tr> &args) { + GemvBatched(const GemmArgs &args) { /* Just create a subgemm with batches->M */ - GemmArgs<Tr> newargs = args; + GemmArgs newargs = args; newargs._Msize = args._nbatches; newargs._nbatches = 1; newargs._cfg = nullptr; @@ -47,13 +47,15 @@ public: void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, const int B_multi_stride, - Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override { - UNUSED(lda); - UNUSED(ldc); + Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const Tr *bias, const int bias_multi_stride) override { /* A and C's batch stride becomes their new row stride. New batch stride is 0 as nbatches for subgemm is always 1. */ _subgemm->set_arrays(A, A_batch_stride, 0, A_multi_stride, B, ldb, B_multi_stride, - C, C_batch_stride, 0, C_multi_stride); + C, C_batch_stride, 0, C_multi_stride, + bias, bias_multi_stride); + UNUSED(lda); + UNUSED(ldc); } unsigned int get_window_size() const override { diff --git a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp index 55b1f9bbe6..49681ec404 100644 --- a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp @@ -53,7 +53,7 @@ class GemvNativeTransposed : public GemmCommon<To, Tr> { const unsigned int _nmultis; - const Tr _beta; + const Activation _act; const CPUInfo * const _ci; @@ -64,8 +64,8 @@ public: GemvNativeTransposed(GemvNativeTransposed &) = delete; GemvNativeTransposed & operator= (GemvNativeTransposed &) = delete; - GemvNativeTransposed(const GemmArgs<Tr> &args) - : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _beta(args._beta), _ci(args._ci) { + GemvNativeTransposed(const GemmArgs &args) + : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _act(args._act), _ci(args._ci) { /* For now don't do any blocking. TODO: figure out if we should. */ m_block = _Ksize; n_block = _Nsize; @@ -111,7 +111,18 @@ public: strat.kernel(this->_Bptr + (multi * this->_B_multi_stride) + (m0 * this->_ldb) + n0, this->_Aptr + (multi * this->_A_multi_stride) + m0, this->_Cptr + (multi * this->_C_multi_stride) + n0, - _beta, this->_ldb, (mmax-m0), (nmax-n0)); + static_cast<Tr>(0), this->_ldb, (mmax-m0), (nmax-n0)); + + // Handle activation separately for now + if (this->_bias) { + activator<true>(this->_Cptr + (multi * this->_C_multi_stride) + n0, 0, + this->_bias + (multi * this->_bias_multi_stride) + n0, + _act, 1, (nmax-n0)); + } else { + activator<false>(this->_Cptr + (multi * this->_C_multi_stride) + n0, 0, + static_cast<const Tr *>(nullptr), + _act, 1, (nmax-n0)); + } } } } diff --git a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp index 92064180a2..26fdfba8ff 100644 --- a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp @@ -26,7 +26,7 @@ #include <stdio.h> #include "arm_gemm.hpp" - +#include "bias_adder.hpp" #include "mergeresults.hpp" #include "transform.hpp" @@ -53,7 +53,7 @@ class GemvPretransposed : public GemmCommon<To, Tr> { const bool _trB; - const Tr _beta; + const Activation _act; const CPUInfo * const _ci; @@ -68,8 +68,8 @@ public: GemvPretransposed(GemvPretransposed &) = delete; GemvPretransposed & operator= (GemvPretransposed &) = delete; - GemvPretransposed(const GemmArgs<Tr> &args) - : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _beta(args._beta), _ci(args._ci), + GemvPretransposed(const GemmArgs &args) + : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _act(args._act), _ci(args._ci), _buffer_per_multi(_Ksize * iceildiv(_Nsize, strategy::A_interleave()) * strategy::A_interleave()) { /* For now don't do any blocking. TODO: figure out if we should. */ if (args._cfg && args._cfg->inner_block_size) { @@ -128,7 +128,18 @@ public: (_Ksize * strategy::A_interleave()), this->_Aptr + (multi * this->_A_multi_stride) + m0, this->_Cptr + (multi * this->_C_multi_stride) + n, - _beta, (mmax-m0), (nmax-n)); + static_cast<Tr>(0), (mmax-m0), (nmax-n)); + + // Handle activation separately for now + if (this->_bias) { + activator<true>(this->_Cptr + (multi * this->_C_multi_stride) + n, 0, + this->_bias + (multi * this->_bias_multi_stride) + n, + _act, 1, (nmax-n)); + } else { + activator<false>(this->_Cptr + (multi * this->_C_multi_stride) + n, 0, + static_cast<const Tr *>(nullptr), + _act, 1, (nmax-n)); + } } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp index cc205dc6e3..941281f24d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp @@ -61,7 +61,9 @@ public: kern_type kernel = a64_gemm_s8_12x8; gemm_s8_12x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto mod = ci->get_cpu_model(); + + if (mod == CPUModel::A55r1) { kernel = a64_gemm_s8_12x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp index 9032ba67b3..c228b8b858 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp @@ -25,8 +25,6 @@ #ifdef __aarch64__ -#include "arm_gemm.hpp" - #include "../std_transforms_fixed.hpp" namespace arm_gemm { @@ -71,7 +69,9 @@ public: kern_type kernel = a64_gemm_u8_12x8; gemm_u8_12x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto mod = ci->get_cpu_model(); + + if (mod == CPUModel::A55r1) { kernel = a64_gemm_u8_12x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp index 5b850b7a20..283fea1102 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp @@ -25,8 +25,6 @@ #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) -#include "arm_gemm.hpp" - #include "../std_transforms_fixed.hpp" namespace arm_gemm { @@ -66,7 +64,9 @@ public: kern_type kernel = a64_hgemm_asimd_24x8; hgemm_24x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto model = ci->get_cpu_model(); + + if (model == CPUModel::A55r1) { kernel = a64_hgemm_asimd_24x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp index 560593958a..1ce934d413 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, float, int, int, int); -void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, float, int, int, int); +void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); +void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); class hybrid_fp32_mla_16x4 { @@ -41,10 +41,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsFixed<operand_type, result_type, 4, 16, 1> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp index 7261761d7e..e3325f7728 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp @@ -25,20 +25,41 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); @@ -48,7 +69,6 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -58,7 +78,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float float result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float); float *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -68,6 +88,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float if (use_result_buffer) { c_ptr0 = result_buffer; } + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -76,17 +97,17 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -95,21 +116,16 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -434,6 +450,16 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v19.4s, v11.4s, v0.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -444,7 +470,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -458,56 +484,47 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "temploadreg3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -931,9 +948,27 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v23.4s, v11.4s, v1.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" "str q20, [c_ptr1]\n" @@ -947,7 +982,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -965,131 +1000,118 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v24.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v25.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "movi v26.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "movi v27.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ins v14.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" "fmla v16.4s, v8.4s, v0.s[0]\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" "fmla v20.4s, v8.4s, v1.s[0]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "fmla v24.4s, v8.4s, v2.s[0]\n" - "ldr d4, [%[a_ptr0]]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "fmla v17.4s, v9.4s, v0.s[0]\n" - "ldr temploadreg0, [%[a_ptr0], #0x8]\n" + "ldr d4, [%[a_ptr0]]\n" "fmla v21.4s, v9.4s, v1.s[0]\n" - "ldr d5, [a_ptr1]\n" + "ldr temploadreg0, [%[a_ptr0], #0x8]\n" "fmla v25.4s, v9.4s, v2.s[0]\n" - "ldr temploadreg1, [a_ptr1, #0x8]\n" + "ldr d5, [a_ptr1]\n" "fmla v18.4s, v10.4s, v0.s[0]\n" - "ldr d6, [a_ptr2]\n" + "ldr temploadreg1, [a_ptr1, #0x8]\n" "fmla v22.4s, v10.4s, v1.s[0]\n" - "ldr temploadreg2, [a_ptr2, #0x8]\n" + "ldr d6, [a_ptr2]\n" "fmla v26.4s, v10.4s, v2.s[0]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg2, [a_ptr2, #0x8]\n" "fmla v19.4s, v11.4s, v0.s[0]\n" - "ins v4.d[1], temploadreg0\n" + "ldr d8, [%[b_ptr0]]\n" "fmla v23.4s, v11.4s, v1.s[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ins v4.d[1], temploadreg0\n" "fmla v27.4s, v11.4s, v2.s[0]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "fmla v16.4s, v12.4s, v0.s[1]\n" - "ins v5.d[1], temploadreg1\n" + "ldr d9, [%[b_ptr0], #0x10]\n" "fmla v20.4s, v12.4s, v1.s[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ins v5.d[1], temploadreg1\n" "fmla v24.4s, v12.4s, v2.s[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "fmla v17.4s, v13.4s, v0.s[1]\n" - "ins v6.d[1], temploadreg2\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "fmla v21.4s, v13.4s, v1.s[1]\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "ins v6.d[1], temploadreg2\n" "fmla v25.4s, v13.4s, v2.s[1]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "fmla v18.4s, v14.4s, v0.s[1]\n" - "ins v15.d[1], temploadreg3\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "fmla v22.4s, v14.4s, v1.s[1]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ins v15.d[1], temploadreg3\n" "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" "fmla v19.4s, v15.4s, v0.s[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x48]\n" + "ins v8.d[1], temploadreg0\n" "fmla v23.4s, v15.4s, v1.s[1]\n" - "ldr d13, [%[b_ptr0], #0x50]\n" + "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" "fmla v16.4s, v8.4s, v0.s[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "fmla v20.4s, v8.4s, v1.s[2]\n" @@ -1113,41 +1135,39 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v23.4s, v11.4s, v1.s[2]\n" "ins v15.d[1], temploadreg3\n" "fmla v27.4s, v11.4s, v2.s[2]\n" - "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - "fmla v16.4s, v12.4s, v0.s[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" "ins v8.d[1], temploadreg0\n" - "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" + "ins v9.d[1], temploadreg1\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla v16.4s, v8.4s, v4.s[0]\n" - "ins v9.d[1], temploadreg1\n" - "fmla v20.4s, v8.4s, v5.s[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" "ins v10.d[1], temploadreg2\n" - "add a_ptr1, a_ptr1, #0x20\n" "fmla v17.4s, v9.4s, v4.s[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "fmla v21.4s, v9.4s, v5.s[0]\n" @@ -1163,70 +1183,71 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v19.4s, v11.4s, v4.s[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "fmla v23.4s, v11.4s, v5.s[0]\n" - "ldr d1, [a_ptr1, #-0x10]\n" - "fmla v27.4s, v11.4s, v6.s[0]\n" "ins v13.d[1], temploadreg1\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ins v14.d[1], temploadreg2\n" "fmla v16.4s, v12.4s, v4.s[1]\n" - "ldr temploadreg1, [a_ptr1, #-0x8]\n" + "ldr d8, [%[b_ptr0]]\n" "fmla v20.4s, v12.4s, v5.s[1]\n" - "ins v14.d[1], temploadreg2\n" + "ins v0.d[1], temploadreg0\n" "fmla v24.4s, v12.4s, v6.s[1]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "fmla v17.4s, v13.4s, v4.s[1]\n" - "ins v0.d[1], temploadreg0\n" + "ldr d9, [%[b_ptr0], #0x10]\n" "fmla v21.4s, v13.4s, v5.s[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "fmla v25.4s, v13.4s, v6.s[1]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "fmla v18.4s, v14.4s, v4.s[1]\n" - "ins v1.d[1], temploadreg1\n" + "ins v15.d[1], temploadreg3\n" "fmla v22.4s, v14.4s, v5.s[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "fmla v26.4s, v14.4s, v6.s[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x20\n" - "ins v15.d[1], temploadreg3\n" - "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - "ldr d2, [a_ptr2, #-0x10]\n" - "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "ldr d12, [%[b_ptr0], #0x40]\n" + "ins v8.d[1], temploadreg0\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla v19.4s, v15.4s, v4.s[1]\n" - "ldr temploadreg2, [a_ptr2, #-0x8]\n" + "ldr d1, [a_ptr1, #-0x10]\n" "fmla v23.4s, v15.4s, v5.s[1]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ldr temploadreg1, [a_ptr1, #-0x8]\n" "fmla v27.4s, v15.4s, v6.s[1]\n" - "ldr d12, [%[b_ptr0], #0x40]\n" - "ins v8.d[1], temploadreg0\n" - "ins v2.d[1], temploadreg2\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "fmla v16.4s, v8.4s, v4.s[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "fmla v20.4s, v8.4s, v5.s[2]\n" - "ins v9.d[1], temploadreg1\n" + "ins v1.d[1], temploadreg1\n" "fmla v24.4s, v8.4s, v6.s[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x58]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "ins v10.d[1], temploadreg2\n" - "fmla v17.4s, v9.4s, v4.s[2]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "fmla v21.4s, v9.4s, v5.s[2]\n" + "add a_ptr2, a_ptr2, #0x20\n" "ldr d15, [%[b_ptr0], #0x70]\n" - "fmla v25.4s, v9.4s, v6.s[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "ldr d2, [a_ptr2, #-0x10]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "ldr temploadreg2, [a_ptr2, #-0x8]\n" + "ins v9.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ins v11.d[1], temploadreg3\n" - "fmla v18.4s, v10.4s, v4.s[2]\n" + "ins v2.d[1], temploadreg2\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" "ins v12.d[1], temploadreg0\n" - "fmla v26.4s, v10.4s, v6.s[2]\n" - "ins v13.d[1], temploadreg1\n" "fmla v19.4s, v11.4s, v4.s[2]\n" - "ins v14.d[1], temploadreg2\n" + "ins v10.d[1], temploadreg2\n" "fmla v23.4s, v11.4s, v5.s[2]\n" - "ins v15.d[1], temploadreg3\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "fmla v27.4s, v11.4s, v6.s[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x100\n" + "ins v13.d[1], temploadreg1\n" "fmla v16.4s, v12.4s, v4.s[3]\n" + "ins v15.d[1], temploadreg3\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ins v14.d[1], temploadreg2\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x100\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "fmla v20.4s, v12.4s, v5.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" @@ -1258,13 +1279,13 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "ins v14.d[1], temploadreg2\n" "b.ne 3b\n" "2:\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [c_ptr1]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" @@ -1554,13 +1575,39 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v27.4s, v11.4s, v2.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" "str q22, [c_ptr1, #0x20]\n" "str q23, [c_ptr1, #0x30]\n" "str q24, [c_ptr2]\n" @@ -1576,7 +1623,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1599,89 +1646,72 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q3, [a_ptr3]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v24.4s, #0\n" + "mov v28.16b, v16.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v25.4s, #0\n" + "mov v29.16b, v17.16b\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v26.4s, #0\n" + "mov v30.16b, v18.16b\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "movi v27.4s, #0\n" + "mov v31.16b, v19.16b\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "movi v28.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v29.4s, #0\n" - "ins v14.d[1], temploadreg2\n" - "movi v30.4s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "movi v31.4s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" + "ins v14.d[1], temploadreg2\n" "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "fmul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "fmul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "fmul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "fmul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -2303,16 +2333,50 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v31.4s, v11.4s, v3.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" "str q25, [c_ptr2, #0x10]\n" "str q26, [c_ptr2, #0x20]\n" "str q27, [c_ptr2, #0x30]\n" @@ -2331,7 +2395,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp index 504769b9f0..d60d08a672 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp @@ -25,20 +25,41 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); @@ -48,7 +69,6 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -58,7 +78,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, float result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float); float *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -68,21 +88,22 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, if (use_result_buffer) { c_ptr0 = result_buffer; } + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: __asm __volatile ( - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -90,21 +111,16 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -291,13 +307,23 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v19.4s, v11.4s, v0.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -307,55 +333,46 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "c_ptr1 .req X1\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -635,9 +652,27 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v23.4s, v11.4s, v1.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" "str q20, [c_ptr1]\n" @@ -647,7 +682,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -661,71 +696,58 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v24.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v25.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" - "movi v26.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v27.4s, #0\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -1098,13 +1120,39 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v27.4s, v11.4s, v2.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" "str q22, [c_ptr1, #0x20]\n" "str q23, [c_ptr1, #0x30]\n" "str q24, [c_ptr2]\n" @@ -1116,7 +1164,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1135,87 +1183,70 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q3, [a_ptr3]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v24.4s, #0\n" + "mov v28.16b, v16.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v25.4s, #0\n" + "mov v29.16b, v17.16b\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v26.4s, #0\n" + "mov v30.16b, v18.16b\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "movi v27.4s, #0\n" + "mov v31.16b, v19.16b\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v28.4s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "movi v29.4s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "movi v30.4s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "fmul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "fmul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "fmul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "fmul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -1681,16 +1712,50 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v31.4s, v11.4s, v3.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" "str q25, [c_ptr2, #0x10]\n" "str q26, [c_ptr2, #0x20]\n" "str q27, [c_ptr2, #0x30]\n" @@ -1705,7 +1770,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp index c8934dff8a..5a6fabcfa9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class hybrid_s8s32_dot_16x4 { @@ -41,10 +41,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp index 3f72e4bea1..3ecf0151aa 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in int32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t); int32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -78,7 +80,6 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -94,44 +95,21 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr d14, [%[b_ptr0], #0x60]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q10, [%[b_ptr0], #0x20]\n" - "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "subs %[loops], %[loops], #0x1\n" @@ -146,108 +124,108 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" @@ -258,29 +236,29 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ins v14.d[1], temploadreg2\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" @@ -291,137 +269,137 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -430,30 +408,30 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -464,7 +442,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -476,178 +454,144 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X3\n" "temploadreg2 .req X4\n" "temploadreg3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" + "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "movi v20.4s, #0\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "movi v21.4s, #0\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" + "movi v22.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" + "movi v23.4s, #0\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ins v14.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "subs %[loops], %[loops], #0x1\n" "ldr d11, [%[b_ptr0], #0x30]\n" - "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" "ins v15.d[1], temploadreg3\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "add a_ptr1, a_ptr1, #0x20\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v1.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" @@ -656,55 +600,55 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -714,223 +658,223 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "b.ne 3b\n" - "2:\n" "ins v14.d[1], temploadreg2\n" - "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr d13, [%[b_ptr0], #0x50]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -939,43 +883,43 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -992,7 +936,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -1006,228 +950,183 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X5\n" "temploadreg2 .req X6\n" "temploadreg3 .req X7\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v25.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v26.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ldr q1, [a_ptr1]\n" "movi v27.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" "ins v14.d[1], temploadreg2\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x20\n" @@ -1235,65 +1134,65 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d2, [a_ptr2, #-0x10]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ins v2.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -1304,275 +1203,275 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1581,56 +1480,56 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1653,7 +1552,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1670,364 +1569,309 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X7\n" "temploadreg2 .req X8\n" "temploadreg3 .req X9\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q3, [a_ptr3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v25.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v26.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr q1, [a_ptr1]\n" "movi v27.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ins v14.d[1], temploadreg2\n" "movi v28.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v29.4s, #0\n" - "ins v14.d[1], temploadreg2\n" + "ldr q2, [a_ptr2]\n" "movi v30.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr3, a_ptr2, %[lda]\n" "movi v31.4s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add a_ptr3, a_ptr3, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" - "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" - "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" "ldr q3, [a_ptr3]\n" - "ldr q8, [%[b_ptr0]]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr3, a_ptr3, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr d2, [a_ptr2, #-0x10]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ins v2.d[1], temploadreg2\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr d3, [a_ptr3, #-0x10]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" "ldr temploadreg3, [a_ptr3, #-0x8]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ins v3.d[1], temploadreg3\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" "ins v14.d[1], temploadreg2\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" @@ -2035,319 +1879,319 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -2356,69 +2200,69 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -2447,7 +2291,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp index 1ad924612e..b48b674621 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ int32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t); int32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -74,7 +76,6 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ switch(M-y) { case 1: __asm __volatile ( - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -89,196 +90,174 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q10, [%[b_ptr0], #0x20]\n" - "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -287,37 +266,37 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -325,316 +304,283 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" + "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "movi v20.4s, #0\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "movi v21.4s, #0\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" + "movi v22.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" + "movi v23.4s, #0\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -643,43 +589,43 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -692,7 +638,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -702,421 +648,378 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v25.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" + "ldr q1, [a_ptr1]\n" "movi v26.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v27.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1125,56 +1028,56 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1193,7 +1096,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1206,526 +1109,473 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q3, [a_ptr3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v25.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr q1, [a_ptr1]\n" "movi v26.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v27.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q2, [a_ptr2]\n" "movi v28.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr3, a_ptr2, %[lda]\n" "movi v29.4s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "ldr q3, [a_ptr3]\n" "movi v30.4s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "movi v31.4s, #0\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" - "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" - "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" - "ldr q3, [a_ptr3]\n" - "ldr q8, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr3, a_ptr3, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q3, [a_ptr3, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1734,69 +1584,69 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1821,7 +1671,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp index 48731efc57..5295650e7b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class hybrid_u8u32_dot_16x4 { @@ -41,7 +41,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp index e343d58fa0..e8ed0c311e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t); uint32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -78,7 +80,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -97,21 +99,16 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -119,19 +116,19 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "subs %[loops], %[loops], #0x1\n" @@ -146,108 +143,108 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" @@ -266,21 +263,21 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" @@ -291,134 +288,134 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -430,10 +427,10 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -449,10 +446,10 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -464,7 +461,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -478,7 +475,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "temploadreg3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -503,31 +500,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -535,33 +523,33 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "subs %[loops], %[loops], #0x1\n" @@ -571,83 +559,83 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x20\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v1.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" @@ -656,55 +644,55 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -723,211 +711,211 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr d13, [%[b_ptr0], #0x50]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -939,16 +927,16 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -967,14 +955,14 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -992,7 +980,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -1010,7 +998,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1042,41 +1030,28 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -1085,149 +1060,149 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x20\n" @@ -1235,65 +1210,65 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d2, [a_ptr2, #-0x10]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ins v2.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -1313,263 +1288,263 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1581,22 +1556,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1618,18 +1593,18 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1653,7 +1628,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1676,7 +1651,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1714,51 +1689,34 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -1767,261 +1725,261 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr d2, [a_ptr2, #-0x10]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ins v2.d[1], temploadreg2\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr d3, [a_ptr3, #-0x10]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" "ldr temploadreg3, [a_ptr3, #-0x8]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ins v3.d[1], temploadreg3\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" @@ -2036,315 +1994,315 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -2356,28 +2314,28 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -2402,22 +2360,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -2447,7 +2405,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp index 6301fa5657..23d919a64c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint uint32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t); uint32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -74,7 +76,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint switch(M-y) { case 1: __asm __volatile ( - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -92,93 +94,88 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" @@ -189,93 +186,93 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -287,10 +284,10 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -306,10 +303,10 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -317,7 +314,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -327,7 +324,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "c_ptr1 .req X1\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -351,287 +348,278 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -643,16 +631,16 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -671,14 +659,14 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -692,7 +680,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -706,7 +694,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -736,190 +724,177 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" @@ -927,193 +902,193 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1125,22 +1100,22 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1162,18 +1137,18 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1193,7 +1168,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1212,7 +1187,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1248,236 +1223,219 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q3, [a_ptr3, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" @@ -1486,243 +1444,243 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1734,28 +1692,28 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1780,22 +1738,22 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1821,7 +1779,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp index 06622d6f2e..a86e8ec068 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,51 +23,61 @@ */ #pragma once -#ifdef __ARM_FEATURE_SVE - - +#ifdef __aarch64__ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int); +void a64_native_fp32_mla_16x4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); -class smallK_fp32_mla_1VLx4 +class native_fp32_mla_16x4 { public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } static unsigned int out_width() { - return get_vector_length<float>() * 1; + return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } - - // Default to the generic kernel - kern_type kernel=sve_smallK_fp32_mla_1VLx4; - - smallK_fp32_mla_1VLx4(const CPUInfo *ci) + static constexpr bool supports_bias() { + return true; + } + static constexpr bool supports_activation() + { + return true; } + + + + // Default to the generic kernel + kern_type kernel=a64_native_fp32_mla_16x4; + + native_fp32_mla_16x4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp new file mode 100644 index 0000000000..82e7333ee3 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp @@ -0,0 +1,1708 @@ +/* + * Copyright (c) 2018-2019 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include <algorithm> + +#include "arm_gemm.hpp" + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_native_fp32_mla_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { + const long loops_count = ((K + 4) / 8) - 1; + K -= loops_count * 8; + const long regs_count = (K / 4) - 1; + K -= (regs_count + 1) * 4; + const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y=0; y<M; y+=4) { + const float * const a_ptr0_base = A + (y * lda); + const unsigned long ldab = lda * sizeof(float); + + float *c_ptr0 = C + (y * ldc); + + for (int x0=0; x0<N; x0+=16ul) { + const long width = std::min((unsigned long)N-x0, 16ul); + long loops = loops_count; + long regs = regs_count; + long blocks = blocks_count; + const float *a_ptr0 = a_ptr0_base; + const float *b_ptr0 = B + x0; + long ldbb = ldb * sizeof(float); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; + + switch(M-y) { + case 1: + __asm __volatile ( + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "ldr q0, [%[a_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q8, [%[b_ptr0]]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q4, [%[a_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "str q17, [%[c_ptr0], #0x10]\n" + "str q18, [%[c_ptr0], #0x20]\n" + "str q19, [%[c_ptr0], #0x30]\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 2: + __asm __volatile ( + "a_ptr1 .req X0\n" + "c_ptr1 .req X1\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "ldr q1, [a_ptr1]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov v22.16b, v18.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v23.16b, v19.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "str q20, [c_ptr1]\n" + "str q21, [c_ptr1, #0x10]\n" + "str q22, [c_ptr1, #0x20]\n" + "str q23, [c_ptr1, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq c_ptr1\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" + ); + break; + case 3: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "c_ptr1 .req X2\n" + "c_ptr2 .req X3\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "mov v24.16b, v16.16b\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "mov v25.16b, v17.16b\n" + "ldr q1, [a_ptr1]\n" + "mov v22.16b, v18.16b\n" + "ldr q2, [a_ptr2]\n" + "mov v23.16b, v19.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v26.16b, v18.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "mov v27.16b, v19.16b\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add a_ptr2, a_ptr2, #0x20\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q2, [a_ptr2, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [c_ptr2]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr s2, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x4\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" + "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" + "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "str q22, [c_ptr1, #0x20]\n" + "str q23, [c_ptr1, #0x30]\n" + "str q24, [c_ptr2]\n" + "str q25, [c_ptr2, #0x10]\n" + "str q26, [c_ptr2, #0x20]\n" + "str q27, [c_ptr2, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" + ); + break; + default: + case 4: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "c_ptr1 .req X3\n" + "c_ptr2 .req X4\n" + "c_ptr3 .req X5\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "mov v24.16b, v16.16b\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "mov v25.16b, v17.16b\n" + "ldr q1, [a_ptr1]\n" + "mov v22.16b, v18.16b\n" + "ldr q2, [a_ptr2]\n" + "mov v23.16b, v19.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v26.16b, v18.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "mov v27.16b, v19.16b\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "mov v28.16b, v16.16b\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "mov v29.16b, v17.16b\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "mov v30.16b, v18.16b\n" + "ldr q3, [a_ptr3]\n" + "mov v31.16b, v19.16b\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add a_ptr3, a_ptr3, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q7, [a_ptr3]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "add a_ptr2, a_ptr2, #0x20\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x20\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q2, [a_ptr2, #-0x10]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q3, [a_ptr3, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v28.4s, v8.4s, v7.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "fmla v29.4s, v9.4s, v7.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "fmla v30.4s, v10.4s, v7.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "fmla v31.4s, v11.4s, v7.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "fmla v28.4s, v12.4s, v7.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "fmla v29.4s, v13.4s, v7.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "fmla v30.4s, v14.4s, v7.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "fmla v31.4s, v15.4s, v7.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v28.4s, v8.4s, v7.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v29.4s, v9.4s, v7.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v30.4s, v10.4s, v7.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v31.4s, v11.4s, v7.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v28.4s, v12.4s, v7.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v29.4s, v13.4s, v7.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v30.4s, v14.4s, v7.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "fmla v31.4s, v15.4s, v7.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [c_ptr2]\n" + "prfm PSTL1KEEP, [c_ptr3]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q7, [a_ptr3]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x10\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v28.4s, v8.4s, v7.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "fmla v29.4s, v9.4s, v7.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "fmla v30.4s, v10.4s, v7.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "fmla v31.4s, v11.4s, v7.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "fmla v28.4s, v12.4s, v7.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "fmla v29.4s, v13.4s, v7.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "fmla v30.4s, v14.4s, v7.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "fmla v31.4s, v15.4s, v7.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v28.4s, v8.4s, v7.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v29.4s, v9.4s, v7.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v30.4s, v10.4s, v7.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v31.4s, v11.4s, v7.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v28.4s, v12.4s, v7.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v29.4s, v13.4s, v7.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v30.4s, v14.4s, v7.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "fmla v31.4s, v15.4s, v7.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr s2, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x4\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr s3, [a_ptr3]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x4\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" + "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" + "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" + "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "str q25, [c_ptr2, #0x10]\n" + "str q26, [c_ptr2, #0x20]\n" + "str q27, [c_ptr2, #0x30]\n" + "str q28, [c_ptr3]\n" + "str q29, [c_ptr3, #0x10]\n" + "str q30, [c_ptr3, #0x20]\n" + "str q31, [c_ptr3, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" + ); + break; + } + + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp deleted file mode 100644 index 9b70827125..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -namespace arm_gemm { - -// Actual kernel implementations -void a64_sgemm_nativeA_pretransposeB_16x4(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int); - -// Native A/Pretranspose B SGEMM "strategy" class. -// -// This describes the characteristics of a family of kernels, in terms of -// the required interleave properties and the output block size. -// -// All kernels in the family must share these characteristics. The actual -// kernel to be used can be chosen at runtime, based on the CPUInfo -// structure. -class sgemm_nativeA_pretransposeB_16x4 { -public: - typedef float operand_type; - typedef float result_type; - - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int); - - /* Desired data layout for B buffer (used for pretranspose) */ - static const int B_interleave = 16; - static const int B_block = 1; - static const bool B_transpose = true; - - /* Kernel blocking parameters */ - static unsigned int out_width() { - return 16; - } - - static unsigned int out_height() { - return 4; - } - - static unsigned int k_unroll() { - return 1; - } - - StdTransformsFixed<operand_type, result_type, 4, 16> transforms = {}; - - // Default to the generic kernel - kern_type kernel=a64_sgemm_nativeA_pretransposeB_16x4; - - sgemm_nativeA_pretransposeB_16x4(const CPUInfo *ci) { - UNUSED(ci); - } -}; - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp deleted file mode 100644 index c51c8ff863..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp +++ /dev/null @@ -1,970 +0,0 @@ -/* - * Copyright (c) 2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __aarch64__ - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -#include <algorithm> -#include <cstddef> -#include <cstring> - -#include <arm_neon.h> - -namespace arm_gemm { - -void a64_sgemm_nativeA_pretransposeB_16x4(const float *A, int lda, const float *B_panel, float *C, int ldc, float beta, unsigned int numrows, unsigned int numcols, unsigned int K) { - const bool oddk = ((K % 8) >= 4); - const bool beta0 = (beta == 0.0f); - const unsigned int oddones = (K % 4); - - /* Use some small temporary arrays to cope with "ragged" M/N sizes. - * - * "dummy_A_buf" is used to avoid overreading the A input for ragged M, - * and also for output if N is not ragged. - * - * Since the B input is pretransposed it will be padded as needed, so no - * need to worry about overreading that. - * - * "C_buf" is used to avoid overreading or overwriting the output for - * ragged N cases. - */ - float dummy_A_buf[16]; - float C_buf[64]; - - std::memset(dummy_A_buf, 0, sizeof(dummy_A_buf)); - std::memset(C_buf, 0, sizeof(C_buf)); - - for (unsigned int y=0; y<numrows; y+=4) { - const float *b_ptr = B_panel; - const unsigned int active_rows = std::min(numrows - y, 4U); - - /* Increment values to be used to advance A pointers - these get set - * to zero when the corresponding row isn't being used due to ragged - * M, so it will just read the dummy buffer repeatedly. Values are - * in bytes (8x sizeof(float)). */ - const unsigned long a_incr1 = (active_rows > 1) ? 32 : 0; - const unsigned long a_incr2 = (active_rows > 2) ? 32 : 0; - const unsigned long a_incr3 = (active_rows > 3) ? 32 : 0; - - /* Starting points for A pointers on this loop */ - const float * const a_ptr0_base = A + (y * lda); - const float * const a_ptr1_base = (active_rows > 1) ? (a_ptr0_base + lda) : dummy_A_buf; - const float * const a_ptr2_base = (active_rows > 2) ? (a_ptr1_base + lda) : dummy_A_buf; - const float * const a_ptr3_base = (active_rows > 3) ? (a_ptr2_base + lda) : dummy_A_buf; - - /* Starting points for C pointers on this loop */ - float *c_ptr0 = C + (y * ldc); - float *c_ptr1 = (active_rows > 1) ? (c_ptr0 + ldc) : dummy_A_buf; - float *c_ptr2 = (active_rows > 2) ? (c_ptr1 + ldc) : dummy_A_buf; - float *c_ptr3 = (active_rows > 3) ? (c_ptr2 + ldc) : dummy_A_buf; - - for (unsigned int x0=0; x0<numcols; x0+=16) { - const unsigned int active_cols = std::min(numcols - x0, 16U); - const bool use_result_buf = (active_cols < 16); - - /* Reset the A pointers for this loop. */ - const float *a_ptr0 = a_ptr0_base; - const float *a_ptr1 = a_ptr1_base; - const float *a_ptr2 = a_ptr2_base; - const float *a_ptr3 = a_ptr3_base; - - /* Override C pointers if the result buffer is in use. */ - if (use_result_buf) { - c_ptr0 = C_buf; - c_ptr1 = C_buf + 16; - c_ptr2 = C_buf + 32; - c_ptr3 = C_buf + 48; - - /* If beta is non-zero, prepopulate the result buffer */ - if (!beta0) { - for (unsigned int row=0; row<active_rows; row++) { - for (unsigned int col=0; col<active_cols; col++) { - C_buf[row * 16 + col] = C[((y + row) * ldc) + (x0 + col)]; - } - } - } - } - - unsigned int loops = ((K+4)/8) - 1; - unsigned int odds = oddones; - - __asm __volatile ( - "a0 .req v0\n" - "a1 .req v1\n" - "a2 .req v2\n" - "a3 .req v3\n" - "a0a .req v4\n" - "a1a .req v5\n" - "a2a .req v6\n" - "a3a .req v7\n" - "bb0 .req v8\n" - "bb1 .req v9\n" - "bb2 .req v10\n" - "bb3 .req v11\n" - "b0a .req v12\n" - "b1a .req v13\n" - "b2a .req v14\n" - "b3a .req v15\n" - - "a0q .req q0\n" - "a1q .req q1\n" - "a2q .req q2\n" - "a3q .req q3\n" - "a0aq .req q4\n" - "a1aq .req q5\n" - "a2aq .req q6\n" - "a3aq .req q7\n" - "b0q .req q8\n" - "b1q .req q9\n" - "b2q .req q10\n" - "b3q .req q11\n" - "b0aq .req q12\n" - "b1aq .req q13\n" - "b2aq .req q14\n" - "b3aq .req q15\n" - - "movi v16.4s, #0x0\n" - "ldr a0q, [%[a_ptr0]]\n" - "movi v17.4s, #0x0\n" - "ldr b0q, [%[b_ptr]]\n" - "movi v18.4s, #0x0\n" - "ldr b1q, [%[b_ptr], #16]\n" - "movi v19.4s, #0x0\n" - "ldr b2q, [%[b_ptr], #32]\n" - "movi v20.4s, #0x0\n" - "ldr b3q, [%[b_ptr], #48]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "movi v21.4s, #0x0\n" - "ldr a1q, [%[a_ptr1]]\n" - "movi v22.4s, #0x0\n" - "ldr a2q, [%[a_ptr2]]\n" - "movi v23.4s, #0x0\n" - "ldr a3q, [%[a_ptr3]]\n" - "movi v24.4s, #0x0\n" - "ldr b0aq, [%[b_ptr]]\n" - "movi v25.4s, #0x0\n" - "ldr b1aq, [%[b_ptr], #16]\n" - "movi v26.4s, #0x0\n" - "ldr b2aq, [%[b_ptr], #32]\n" - "cbz %w[beta0], 5f\n" - "movi v27.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x40]") - "movi v28.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x80]") - "movi v29.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0xC0]") - "movi v30.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x100]") - "movi v31.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x140]") - ASM_PREFETCH("[%[b_ptr], #0x180]") - ASM_PREFETCH("[%[b_ptr], #0x1C0]") - ASM_PREFETCH("[%[b_ptr], #0x200]") - - // Skip if no complete loops. - "cbz %w[loops], 4f\n" - "b 1f\n" - - // If beta is non-zero, need to load and multiply by beta - "5:\n" - "ld1r {v4.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #16]\n" - "ldr q18, [%[c_ptr0], #32]\n" - "ldr q19, [%[c_ptr0], #48]\n" - - "ldr q20, [%[c_ptr1]]\n" - "fmul v16.4s, v16.4s, v4.4s\n" - "ldr q21, [%[c_ptr1], #16]\n" - "fmul v17.4s, v17.4s, v4.4s\n" - "ldr q22, [%[c_ptr1], #32]\n" - "fmul v18.4s, v18.4s, v4.4s\n" - "ldr q23, [%[c_ptr1], #48]\n" - "fmul v19.4s, v19.4s, v4.4s\n" - - "ldr q24, [%[c_ptr2]]\n" - "fmul v20.4s, v20.4s, v4.4s\n" - "ldr q25, [%[c_ptr2], #16]\n" - "fmul v21.4s, v21.4s, v4.4s\n" - "ldr q26, [%[c_ptr2], #32]\n" - "fmul v22.4s, v22.4s, v4.4s\n" - "ldr q27, [%[c_ptr2], #48]\n" - "fmul v23.4s, v23.4s, v4.4s\n" - - "ldr q28, [%[c_ptr3]]\n" - "fmul v24.4s, v24.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x40]") - "ldr q29, [%[c_ptr3], #16]\n" - "fmul v25.4s, v25.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x80]") - "ldr q30, [%[c_ptr3], #32]\n" - "fmul v26.4s, v26.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0xC0]") - "ldr q31, [%[c_ptr3], #48]\n" - "fmul v27.4s, v27.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x100]") - - "fmul v28.4s, v28.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x140]") - "fmul v29.4s, v29.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x180]") - "fmul v30.4s, v30.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x1C0]") - "fmul v31.4s, v31.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x200]") - - "cbz %w[loops], 4f\n" - - "1:\n" - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - ASM_PREFETCH("[%[b_ptr], #0x240]") - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr], #64]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #80]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #96]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #112]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - ASM_PREFETCH("[%[b_ptr], #0x280]") - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr], #128]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "subs %w[loops], %w[loops], #1\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #144]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #160]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #176]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - ASM_PREFETCH("[%[b_ptr], #0x2C0]") - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr], #192]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #208]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - ASM_PREFETCH("[%[a_ptr0], #0x40]") - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #224]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - ASM_PREFETCH("[%[a_ptr1], #0x40]") - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #240]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], #512\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr], #-256]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - ASM_PREFETCH("[%[b_ptr], #0x100]") - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #-240]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - ASM_PREFETCH("[%[a_ptr2], #0x40]") - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #-224]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr a0q, [%[a_ptr0]]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #-208]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - ASM_PREFETCH("[%[b_ptr], #0x140]") - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr], #-192]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "ldr a1q, [%[a_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #-176]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "ldr a2q, [%[a_ptr2]]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #-160]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "ldr a3q, [%[a_ptr3]]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #-144]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - ASM_PREFETCH("[%[b_ptr], #0x180]") - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr], #-128]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - ASM_PREFETCH("[%[a_ptr3], #0x40]") - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #-112]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #-96]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #-80]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[b_ptr], #0x1C0]") - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - "ldr b0q, [%[b_ptr], #-64]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - "ldr b1q, [%[b_ptr], #-48]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - "ldr b2q, [%[b_ptr], #-32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - "ldr b3q, [%[b_ptr], #-16]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - ASM_PREFETCH("[%[b_ptr], #0x200]") - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "bne 1b\n" - - // Skip to here - "4:\n" - - // Detached final iteration - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr0], #0x40]") - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr1], #0x40]") - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr2], #0x40]") - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr3], #0x40]") - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "cbnz %w[odds], 6f\n" - - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // Odd K case: Just do 4 more. - "2:\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #16\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "add %[a_ptr1], %[a_ptr1], #16\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "add %[a_ptr2], %[a_ptr2], #16\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "add %[a_ptr3], %[a_ptr3], #16\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr0], #0x40]") - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr1], #0x40]") - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr2], #0x40]") - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr3], #0x40]") - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "cbnz %w[odds], 7f\n" - - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // "Odd ones" - lead in from even - "6:\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - "b 8f\n" - - // "Odd ones" - lead in from odd - "7:\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - - // "Odd ones" - loop - "8:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "fmla v28.4s, bb0.4s, a3.4s\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v26.4s, bb2.4s, a2.4s\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "ldr b3q, [%[b_ptr], #48]\n" - "fmla v16.4s, bb0.4s, a0.4s\n" - "bne 8b\n" - - // "Odd ones" - detached final iteration - "9:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.4s\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, bb0.4s, a3.4s\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "str q27, [%[c_ptr2], #48]\n" - - "3:\n" - "str q28, [%[c_ptr3]]\n" - // Increment C pointers for next loop - this looks odd if we - // are using the result buffer, but it's OK as using the - // result buffer implies there will be no next loop. - "add %[c_ptr0], %[c_ptr0], #64\n" - "str q29, [%[c_ptr3], #16]\n" - "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n" - "str q30, [%[c_ptr3], #32]\n" - "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n" - "str q31, [%[c_ptr3], #48]\n" - "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n" - - : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3), - [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds), - [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3) - : [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta), - [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", - "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", - "cc", "memory" - ); - - /* Copy results from result buffer if needed. */ - if (use_result_buf) { - for (unsigned int row=0; row<active_rows; row++) { - for (unsigned int col=0; col<active_cols; col++) { - C[((y + row) * ldc) + (x0 + col)] = C_buf[row * 16 + col]; - } - } - } - } - } -} - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp deleted file mode 100644 index 8325b3f352..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp +++ /dev/null @@ -1,889 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __aarch64__ - -#include <algorithm> -#include <cstddef> -#include <cstring> - -#include <arm_neon.h> - -namespace arm_gemm { - -void a64_sgemm_native_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const int oddk = ((K % 8) >= 4) ? 1 : 0; - const int beta0 = (beta == 0.0f) ? 1 : 0; - const int oddones = (K % 4); - - float dummy_buffer[16]; - - std::memset(dummy_buffer, 0, sizeof(dummy_buffer)); - - /* For now, very naive with no blocking */ - for (int y=0; y<M; y+=4) { - const int activerows = std::min(M-y, 4); - - const float * const a_ptr0_base = A + (y * lda); - const float * const a_ptr1_base = (activerows > 1) ? (a_ptr0_base + lda) : dummy_buffer; - const float * const a_ptr2_base = (activerows > 2) ? (a_ptr1_base + lda) : dummy_buffer; - const float * const a_ptr3_base = (activerows > 3) ? (a_ptr2_base + lda) : dummy_buffer; - - const unsigned long a_incr1 = (activerows > 1) ? 32 : 0; - const unsigned long a_incr2 = (activerows > 2) ? 32 : 0; - const unsigned long a_incr3 = (activerows > 3) ? 32 : 0; - - float *c_ptr0 = C + (y * ldc); - float *c_ptr1 = (activerows > 1) ? c_ptr0 + ldc : dummy_buffer; - float *c_ptr2 = (activerows > 2) ? c_ptr1 + ldc : dummy_buffer; - float *c_ptr3 = (activerows > 3) ? c_ptr2 + ldc : dummy_buffer; - - for (int x0=0; x0<N; x0+=16) { - const float *a_ptr0 = a_ptr0_base; - const float *a_ptr1 = a_ptr1_base; - const float *a_ptr2 = a_ptr2_base; - const float *a_ptr3 = a_ptr3_base; - - const float *b_ptr = B + x0; - - int loops = ((K+4)/8) - 1; - int odds = oddones; - - size_t ldbb = ldb * sizeof(float); - - __asm __volatile ( - "a0 .req v0\n" - "a1 .req v1\n" - "a2 .req v2\n" - "a3 .req v3\n" - "a0a .req v4\n" - "a1a .req v5\n" - "a2a .req v6\n" - "a3a .req v7\n" - "bb0 .req v8\n" - "bb1 .req v9\n" - "bb2 .req v10\n" - "bb3 .req v11\n" - "b0a .req v12\n" - "b1a .req v13\n" - "b2a .req v14\n" - "b3a .req v15\n" - - "a0q .req q0\n" - "a1q .req q1\n" - "a2q .req q2\n" - "a3q .req q3\n" - "a0aq .req q4\n" - "a1aq .req q5\n" - "a2aq .req q6\n" - "a3aq .req q7\n" - "b0q .req q8\n" - "b1q .req q9\n" - "b2q .req q10\n" - "b3q .req q11\n" - "b0aq .req q12\n" - "b1aq .req q13\n" - "b2aq .req q14\n" - "b3aq .req q15\n" - - "movi v16.4s, #0x0\n" - "ldr a0q, [%[a_ptr0]]\n" - "movi v17.4s, #0x0\n" - "ldr b0q, [%[b_ptr]]\n" - "movi v18.4s, #0x0\n" - "ldr b1q, [%[b_ptr], #16]\n" - "movi v19.4s, #0x0\n" - "ldr b2q, [%[b_ptr], #32]\n" - "movi v20.4s, #0x0\n" - "ldr b3q, [%[b_ptr], #48]\n" - "movi v21.4s, #0x0\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "ldr a1q, [%[a_ptr1]]\n" - "movi v22.4s, #0x0\n" - "ldr a2q, [%[a_ptr2]]\n" - "movi v23.4s, #0x0\n" - "ldr a3q, [%[a_ptr3]]\n" - "movi v24.4s, #0x0\n" - "ldr b0aq, [%[b_ptr]]\n" - "movi v25.4s, #0x0\n" - "ldr b1aq, [%[b_ptr], #16]\n" - "movi v26.4s, #0x0\n" - "ldr b2aq, [%[b_ptr], #32]\n" - "cbz %w[beta0], 5f\n" - "movi v27.4s, #0x0\n" - "movi v28.4s, #0x0\n" - "movi v29.4s, #0x0\n" - "movi v30.4s, #0x0\n" - "movi v31.4s, #0x0\n" - - // Skip if no complete loops. - "cbz %w[loops], 4f\n" - "b 1f\n" - - // If beta is non-zero, need to load and multiply by beta - "5:\n" - "ld1r {v4.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #16]\n" - "ldr q18, [%[c_ptr0], #32]\n" - "ldr q19, [%[c_ptr0], #48]\n" - - "ldr q20, [%[c_ptr1]]\n" - "fmul v16.4s, v16.4s, v4.4s\n" - "ldr q21, [%[c_ptr1], #16]\n" - "fmul v17.4s, v17.4s, v4.4s\n" - "ldr q22, [%[c_ptr1], #32]\n" - "fmul v18.4s, v18.4s, v4.4s\n" - "ldr q23, [%[c_ptr1], #48]\n" - "fmul v19.4s, v19.4s, v4.4s\n" - - "ldr q24, [%[c_ptr2]]\n" - "fmul v20.4s, v20.4s, v4.4s\n" - "ldr q25, [%[c_ptr2], #16]\n" - "fmul v21.4s, v21.4s, v4.4s\n" - "ldr q26, [%[c_ptr2], #32]\n" - "fmul v22.4s, v22.4s, v4.4s\n" - "ldr q27, [%[c_ptr2], #48]\n" - "fmul v23.4s, v23.4s, v4.4s\n" - - "ldr q28, [%[c_ptr3]]\n" - "fmul v24.4s, v24.4s, v4.4s\n" - "ldr q29, [%[c_ptr3], #16]\n" - "fmul v25.4s, v25.4s, v4.4s\n" - "ldr q30, [%[c_ptr3], #32]\n" - "fmul v26.4s, v26.4s, v4.4s\n" - "ldr q31, [%[c_ptr3], #48]\n" - "fmul v27.4s, v27.4s, v4.4s\n" - - "fmul v28.4s, v28.4s, v4.4s\n" - "fmul v29.4s, v29.4s, v4.4s\n" - "fmul v30.4s, v30.4s, v4.4s\n" - "fmul v31.4s, v31.4s, v4.4s\n" - - "cbz %w[loops], 4f\n" - - "1:\n" - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "subs %w[loops], %w[loops], #1\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr a0q, [%[a_ptr0]]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "ldr a1q, [%[a_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "ldr a2q, [%[a_ptr2]]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "ldr a3q, [%[a_ptr3]]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "bne 1b\n" - - // Skip to here - "4:\n" - - // Detached final iteration - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "cbnz %w[odds], 6f\n" - - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // Odd K case: Just do 4 more. - "2:\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #16\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "add %[a_ptr1], %[a_ptr1], #16\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "add %[a_ptr2], %[a_ptr2], #16\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "add %[a_ptr3], %[a_ptr3], #16\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "cbnz %w[odds], 7f\n" - - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // "Odd ones" - lead in from even - "6:\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - "b 8f\n" - - // "Odd ones" - lead in from odd - "7:\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - - // "Odd ones" - loop - "8:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "fmla v28.4s, bb0.4s, a3.4s\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v26.4s, bb2.4s, a2.4s\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "ldr b3q, [%[b_ptr], #48]\n" - "fmla v16.4s, bb0.4s, a0.4s\n" - "bne 8b\n" - - // "Odd ones" - detached final iteration - "9:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.4s\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, bb0.4s, a3.4s\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "str q27, [%[c_ptr2], #48]\n" - - "3:\n" - "str q28, [%[c_ptr3]]\n" - "add %[c_ptr0], %[c_ptr0], #64\n" - "str q29, [%[c_ptr3], #16]\n" - "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n" - "str q30, [%[c_ptr3], #32]\n" - "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n" - "str q31, [%[c_ptr3], #48]\n" - "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n" - - : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3), - [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds), - [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3) - : [ldb] "r" (ldbb), [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta), - [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", - "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", - "cc", "memory" - ); - } - } -} - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp index 022efdfc26..352a147282 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -23,51 +23,61 @@ */ #pragma once -#ifdef __ARM_FEATURE_SVE - - +#ifdef __aarch64__ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_fp32_mla_1VLx4(const float *, int, const float *, float *, int, float, int, int, int); +void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); -class smallK_hybrid_fp32_mla_1VLx4 +class smallK_hybrid_fp32_mla_4x6 { public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { - return 4; + return 6; } static unsigned int out_width() { - return get_vector_length<float>() * 1; + return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } - StdTransformsSVE<operand_type, result_type, 4, 1, 1> transforms = {}; - - // Default to the generic kernel - kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx4; + static constexpr bool supports_append() + { + return false; + } - smallK_hybrid_fp32_mla_1VLx4(const CPUInfo *ci) + static constexpr bool supports_bias() { + return true; + } + static constexpr bool supports_activation() + { + return true; } + + StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {}; + + // Default to the generic kernel + kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6; + + smallK_hybrid_fp32_mla_4x6(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp new file mode 100644 index 0000000000..d8e8e52417 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp @@ -0,0 +1,4612 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include <algorithm> + +#include "arm_gemm.hpp" + + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_smallK_hybrid_fp32_mla_4x6(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; + const long ldab = lda * sizeof(float); + const long ldcb = ldc * sizeof(float); + float nullbias[4]; + if (!bias) { + memset(nullbias, 0, (4 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y0=0; y0<M; y0+=6) { + long loops = loops_count; + long oob_rows = std::max(6 - (M-y0), 0); + const float *b_ptr0 = B; + const float *biasptr = bias ? bias : nullbias; + const uint64_t biasinc = bias ? 4*sizeof(float) : 0; + const float *a_ptr0 = A + (y0 * lda); + + float *c_ptr0 = C + (y0 * ldc); + + switch(K) { + case 9: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr s2, [%[a_ptr0]]\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr s5, [a_ptr1]\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr s8, [a_ptr2]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr s11, [a_ptr3]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr s14, [a_ptr4]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr s17, [a_ptr5]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 10: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr d2, [%[a_ptr0]]\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1]\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr d11, [a_ptr3]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr d14, [a_ptr4]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr d17, [a_ptr5]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 11: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr d2, [%[a_ptr0]], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr d17, [a_ptr5], #0x8\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ld1 {v2.s}[2], [%[a_ptr0]]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 12: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q11, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q14, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q17, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q18, [%[b_ptr0]]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 13: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr s3, [%[a_ptr0]]\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr s7, [a_ptr1]\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr s11, [a_ptr2]\n" + "ldr q24, [%[b_ptr0]]\n" + "ldr s15, [a_ptr3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr s19, [a_ptr4]\n" + "ldr s23, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 14: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr d3, [%[a_ptr0]]\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr d7, [a_ptr1]\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr d11, [a_ptr2]\n" + "ldr q24, [%[b_ptr0]]\n" + "ldr d15, [a_ptr3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr d19, [a_ptr4]\n" + "ldr d23, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 15: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr d3, [%[a_ptr0]], #0x8\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr d7, [a_ptr1], #0x8\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr d11, [a_ptr2], #0x8\n" + "ldr q24, [%[b_ptr0]]\n" + "ldr d15, [a_ptr3], #0x8\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr d19, [a_ptr4], #0x8\n" + "ldr d23, [a_ptr5], #0x8\n" + "ld1 {v3.s}[2], [%[a_ptr0]]\n" + "ld1 {v7.s}[2], [a_ptr1]\n" + "ld1 {v11.s}[2], [a_ptr2]\n" + "ld1 {v15.s}[2], [a_ptr3]\n" + "ld1 {v19.s}[2], [a_ptr4]\n" + "ld1 {v23.s}[2], [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + default: + case 16: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr q3, [%[a_ptr0]]\n" + "ldr q7, [a_ptr1]\n" + "ldr q11, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q15, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q19, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q23, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q24, [%[b_ptr0]]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp index a5511931d8..c5d39cbc87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Arm Limited. + * Copyright (c) 2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -25,45 +25,59 @@ #ifdef __aarch64__ -namespace arm_gemm { + + +namespace arm_gemm +{ // Actual kernel implementations -void a64_sgemm_native_16x4(const float *, int, const float *, int, float *, int, float, int, int, int); - -// 12x8 SGEMM "strategy" class. -// -// This describes the characteristics of a family of kernels, in terms of -// the required interleave properties and the output block size. -// -// All kernels in the family must share these characteristics. The actual -// kernel to be used can be chosen at runtime, based on the CPU_type -// structure. -class sgemm_native_16x4 { +void a64_smallK_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); + +class smallK_hybrid_fp32_mla_4x8 +{ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_width() { - return 16; + static constexpr unsigned int out_height() + { + return 8; } - static unsigned int out_height() { + static unsigned int out_width() + { return 4; } - static unsigned int k_unroll() { + static constexpr unsigned int k_unroll() + { return 1; } - // Default to the generic kernel - kern_type kernel=a64_sgemm_native_16x4; + static constexpr bool supports_append() + { + return false; + } - sgemm_native_16x4(const CPUInfo *ci) { - UNUSED(ci); + static constexpr bool supports_bias() + { + return true; } + + static constexpr bool supports_activation() + { + return true; + } + + StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {}; + + // Default to the generic kernel + kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8; + + smallK_hybrid_fp32_mla_4x8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp new file mode 100644 index 0000000000..7ad52059b6 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp @@ -0,0 +1,3340 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include <algorithm> + +#include "arm_gemm.hpp" + + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_smallK_hybrid_fp32_mla_4x8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; + const long ldab = lda * sizeof(float); + const long ldcb = ldc * sizeof(float); + float nullbias[4]; + if (!bias) { + memset(nullbias, 0, (4 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y0=0; y0<M; y0+=8) { + long loops = loops_count; + long oob_rows = std::max(8 - (M-y0), 0); + const float *b_ptr0 = B; + const float *biasptr = bias ? bias : nullbias; + const uint64_t biasinc = bias ? 4*sizeof(float) : 0; + const float *a_ptr0 = A + (y0 * lda); + + float *c_ptr0 = C + (y0 * ldc); + + switch(K) { + case 1: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 2: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d0, [%[a_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 3: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d0, [%[a_ptr0]], #0x8\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" + "ld1 {v0.s}[2], [%[a_ptr0]]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 4: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q1, [a_ptr1]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "ldr q2, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "ldr q3, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "ldr q4, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "ldr q5, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 5: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr s1, [%[a_ptr0]]\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr s3, [a_ptr1]\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr s5, [a_ptr2]\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr s7, [a_ptr3]\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr s9, [a_ptr4]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr s11, [a_ptr5]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr s13, [a_ptr6]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr s15, [a_ptr7]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x50\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x50\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v25.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x50\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 6: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr d1, [%[a_ptr0]]\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr d3, [a_ptr1]\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr d5, [a_ptr2]\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr d7, [a_ptr3]\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr d9, [a_ptr4]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d11, [a_ptr5]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d13, [a_ptr6]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr d15, [a_ptr7]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x60\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x60\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v25.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x60\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v25.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 7: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr d1, [%[a_ptr0]], #0x8\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr d3, [a_ptr1], #0x8\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr d5, [a_ptr2], #0x8\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr d7, [a_ptr3], #0x8\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr d9, [a_ptr4], #0x8\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d11, [a_ptr5], #0x8\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d13, [a_ptr6], #0x8\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr d15, [a_ptr7], #0x8\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "ld1 {v1.s}[2], [%[a_ptr0]]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "ld1 {v3.s}[2], [a_ptr1]\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "ld1 {v5.s}[2], [a_ptr2]\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "ld1 {v7.s}[2], [a_ptr3]\n" + "ld1 {v9.s}[2], [a_ptr4]\n" + "ld1 {v11.s}[2], [a_ptr5]\n" + "ld1 {v13.s}[2], [a_ptr6]\n" + "ld1 {v15.s}[2], [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x70\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x70\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x70\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + default: + case 8: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr q1, [%[a_ptr0]]\n" + "ldr q3, [a_ptr1]\n" + "ldr q5, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q7, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "ldr q9, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "ldr q11, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "ldr q13, [a_ptr6]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "ldr q15, [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "ldr q23, [%[b_ptr0], #0x70]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "ldr q23, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "ldr q23, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp index 7fa02e326e..a1a1fc51af 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_4x6 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 6; } @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp index e8ac2d7489..6d71840801 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -97,335 +98,356 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -493,364 +515,385 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" + "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ins v22.d[1], temploadreg2\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v23.d[1], temploadreg3\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -918,390 +961,408 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ins v23.d[1], temploadreg3\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1369,419 +1430,437 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1914,104 +1993,104 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2023,265 +2102,283 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2420,111 +2517,111 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2535,282 +2632,300 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2949,119 +3064,119 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3073,299 +3188,317 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -3511,126 +3644,126 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3641,316 +3774,334 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp index 64ab6f0fdc..d4082130ad 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -93,313 +94,322 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -459,340 +469,349 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -852,364 +871,370 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1269,391 +1294,397 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1778,344 +1809,362 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q25, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2246,365 +2295,383 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2735,389 +2802,407 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q25, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -3255,410 +3340,428 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp index 6615f1f36b..02be0bd77a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_4x8 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 8; } @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp index 4ed2bc823c..c742fcc890 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -114,47 +115,47 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -172,94 +173,118 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "ins v16.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" - "ins v16.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "subs %[loops], %[loops], #0x1\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr d16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "ins v16.d[1], temploadreg0\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr d16, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" - "ins v16.d[1], temploadreg0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -348,55 +373,55 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -414,23 +439,23 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -441,98 +466,122 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v16.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v17.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v17.d[1], temploadreg1\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -620,56 +669,56 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -687,32 +736,32 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -726,117 +775,141 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v17.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v18.d[1], temploadreg2\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "str q26, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v18.d[1], temploadreg2\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -925,63 +998,63 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -999,41 +1072,41 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1050,136 +1123,160 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v18.d[1], temploadreg2\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v19.d[1], temploadreg3\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v19.d[1], temploadreg3\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1334,50 +1431,50 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1397,155 +1494,179 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v19.d[1], temploadreg3\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v20.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v20.d[1], temploadreg0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1708,59 +1829,59 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1783,174 +1904,198 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v20.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v21.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v21.d[1], temploadreg1\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2113,68 +2258,68 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2200,193 +2345,217 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v21.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v22.d[1], temploadreg2\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v22.d[1], temploadreg2\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2558,77 +2727,77 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2657,212 +2826,236 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v22.d[1], temploadreg2\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v23.d[1], temploadreg3\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" "ins v22.d[1], temploadreg2\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "ldr d23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v23.d[1], temploadreg3\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp index 1eb58e1cd6..2df2c30426 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -110,47 +111,47 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -168,90 +169,114 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - "movi v24.4s, #0\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "movi v24.4s, #0\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -332,55 +357,55 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -398,117 +423,141 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q17, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q17, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -588,56 +637,56 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -655,144 +704,168 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q17, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -873,63 +946,63 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -947,41 +1020,41 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -989,129 +1062,153 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q18, [%[b_ptr0], #0x20]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1258,50 +1355,50 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1310,146 +1407,170 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q19, [%[b_ptr0], #0x30]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1604,59 +1725,59 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1666,163 +1787,187 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q20, [%[b_ptr0], #0x40]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1977,68 +2122,68 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2049,180 +2194,204 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q21, [%[b_ptr0], #0x50]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2386,77 +2555,77 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2468,197 +2637,221 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr q22, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp index adeebf9bbc..1110924e5c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_4x6 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp index 8bf5debc37..ceb4a3b2d3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -97,140 +98,155 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" @@ -238,114 +254,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -354,83 +367,74 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -511,369 +515,372 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" + "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ins v22.d[1], temploadreg2\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v23.d[1], temploadreg3\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -954,161 +961,175 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" @@ -1116,132 +1137,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ins v23.d[1], temploadreg3\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" @@ -1251,98 +1268,88 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1423,174 +1430,188 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" @@ -1598,141 +1619,137 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -1742,105 +1759,95 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1986,104 +1993,104 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2105,142 +2112,142 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" @@ -2252,113 +2259,113 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2510,111 +2517,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2635,152 +2642,152 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" @@ -2792,120 +2799,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3057,119 +3064,119 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3191,161 +3198,161 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" @@ -3357,128 +3364,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3637,126 +3644,126 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3777,171 +3784,171 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" @@ -3953,135 +3960,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp index 1f279afd43..4eecee4c07 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -93,318 +94,309 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -477,345 +469,336 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -888,369 +871,357 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1323,396 +1294,384 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1850,104 +1809,104 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1962,120 +1921,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" @@ -2086,113 +2045,113 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2336,111 +2295,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2455,127 +2414,127 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -2586,120 +2545,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2843,119 +2802,119 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2970,135 +2929,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" @@ -3109,128 +3068,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3381,126 +3340,126 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3515,142 +3474,142 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -3661,135 +3620,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp index a260917d08..fef8909d4f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_4x8 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp index ec29698f48..f30f7e2895 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -114,47 +115,47 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -172,14 +173,14 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -197,33 +198,33 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" @@ -241,32 +242,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -372,55 +373,55 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -438,23 +439,23 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -475,49 +476,49 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" @@ -530,40 +531,40 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -668,56 +669,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -735,32 +736,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -784,61 +785,61 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" "b.ne 8b\n" @@ -850,48 +851,48 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -997,63 +998,63 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -1071,41 +1072,41 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1132,72 +1133,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" "b.ne 8b\n" @@ -1209,56 +1210,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1430,50 +1431,50 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1503,83 +1504,83 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" "b.ne 8b\n" @@ -1591,64 +1592,64 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1828,59 +1829,59 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1913,94 +1914,94 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" "b.ne 8b\n" @@ -2012,72 +2013,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2257,68 +2258,68 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2354,105 +2355,105 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" "b.ne 8b\n" @@ -2464,80 +2465,80 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2726,77 +2727,77 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2835,116 +2836,116 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" "ins v22.d[1], temploadreg2\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "ldr d23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" @@ -2956,88 +2957,88 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp index 46ca013270..16c1d1740f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -110,47 +111,47 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -168,14 +169,14 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -190,33 +191,33 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" @@ -233,32 +234,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -356,55 +357,55 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -422,23 +423,23 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -453,49 +454,49 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -506,40 +507,40 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -636,56 +637,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -703,32 +704,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -744,58 +745,58 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -806,48 +807,48 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -945,63 +946,63 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -1019,41 +1020,41 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1070,67 +1071,67 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1141,56 +1142,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1354,50 +1355,50 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1415,76 +1416,76 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1495,64 +1496,64 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1724,59 +1725,59 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1795,85 +1796,85 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1884,72 +1885,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2121,68 +2122,68 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2202,94 +2203,94 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -2300,80 +2301,80 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2554,77 +2555,77 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2645,103 +2646,103 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr q22, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -2752,88 +2753,88 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp index c6895a6a0e..28ef8071c2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int); +void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); class hybrid_fp16_mla_4VLx4 { @@ -40,10 +40,10 @@ public: typedef __fp16 operand_type; typedef __fp16 result_type; - typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int); + typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<__fp16>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_fp16_mla_4VLx4; - hybrid_fp16_mla_4VLx4(const CPUInfo *ci) - { - - } + hybrid_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp index ab41fb3743..2998f33d87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,37 +25,58 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 8) / 16) - 1; K -= loops_count * 16; const long regs_count = (K / 8) - 1; K -= (regs_count + 1) * 8; const long leftovers = K; + __fp16 nullbias[512]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity()); + const __fp16 * const minptr = &minval; + const __fp16 * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const __fp16 * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(__fp16); __fp16 *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(__fp16); for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>())); - const __fp16 *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = leftovers; const __fp16 *a_ptr0 = a_ptr0_base; const __fp16 *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(__fp16); + const __fp16 *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -69,44 +90,37 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z18.h, #0\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.h, #0\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -322,13 +336,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" - "fmla z17.h, z13.h, z4.h[1]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z4.h[1]\n" + "fmla z17.h, z13.h, z4.h[1]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z4.h[1]\n" + "fmla z18.h, z14.h, z4.h[1]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z4.h[1]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -339,9 +352,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[2]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.h, z13.h, z4.h[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z18.h, z14.h, z4.h[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z19.h, z15.h, z4.h[3]\n" "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" @@ -372,76 +387,77 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[6]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z4.h[7]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z17.h, z13.h, z4.h[7]\n" "fmla z18.h, z14.h, z4.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" "fmla z18.h, z10.h, z0.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" "fmla z18.h, z14.h, z0.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[2]\n" "fmla z18.h, z10.h, z0.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z13.h, z0.h[3]\n" "fmla z18.h, z14.h, z0.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[4]\n" "fmla z18.h, z10.h, z0.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z0.h[5]\n" "fmla z18.h, z14.h, z0.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" "fmla z18.h, z10.h, z0.h[6]\n" @@ -451,33 +467,34 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "fmla z17.h, z9.h, z0.h[2]\n" + "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z18.h, z10.h, z0.h[2]\n" "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z19.h, z11.h, z0.h[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.h, z12.h, z0.h[3]\n" - "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" @@ -504,88 +521,99 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[6]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z0.h[7]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z17.h, z13.h, z0.h[7]\n" "fmla z18.h, z14.h, z0.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[0]\n" "fmla z18.h, z10.h, z4.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z4.h[1]\n" "fmla z18.h, z14.h, z4.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[2]\n" "fmla z18.h, z10.h, z4.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z13.h, z4.h[3]\n" "fmla z18.h, z14.h, z4.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[4]\n" "fmla z18.h, z10.h, z4.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z4.h[5]\n" "fmla z18.h, z14.h, z4.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" "fmla z18.h, z10.h, z4.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -604,60 +632,49 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.h, #0\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.h, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.h, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z23.h, #0\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -974,9 +991,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z20.h, z8.h, z5.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.h, z9.h, z5.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z4.h[0]\n" @@ -1044,6 +1063,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" "fmla z21.h, z9.h, z5.h[6]\n" @@ -1060,13 +1080,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z17.h, z9.h, z0.h[0]\n" "fmla z21.h, z9.h, z1.h[0]\n" @@ -1075,12 +1094,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z17.h, z13.h, z0.h[1]\n" "fmla z21.h, z13.h, z1.h[1]\n" @@ -1089,12 +1108,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z17.h, z9.h, z0.h[2]\n" "fmla z21.h, z9.h, z1.h[2]\n" @@ -1103,12 +1123,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z17.h, z13.h, z0.h[3]\n" "fmla z21.h, z13.h, z1.h[3]\n" @@ -1117,13 +1137,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z17.h, z9.h, z0.h[4]\n" "fmla z21.h, z9.h, z1.h[4]\n" @@ -1132,12 +1151,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z17.h, z13.h, z0.h[5]\n" "fmla z21.h, z13.h, z1.h[5]\n" @@ -1146,10 +1165,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" @@ -1163,19 +1183,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" - "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z5.h, p6/z, [a_ptr1]\n" + "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" + "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z20.h, z12.h, z1.h[1]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -1237,6 +1259,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" "fmla z21.h, z9.h, z1.h[6]\n" @@ -1253,13 +1276,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z17.h, z9.h, z4.h[0]\n" "fmla z21.h, z9.h, z5.h[0]\n" @@ -1268,12 +1290,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z17.h, z13.h, z4.h[1]\n" "fmla z21.h, z13.h, z5.h[1]\n" @@ -1282,12 +1304,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z17.h, z9.h, z4.h[2]\n" "fmla z21.h, z9.h, z5.h[2]\n" @@ -1296,12 +1319,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z17.h, z13.h, z4.h[3]\n" "fmla z21.h, z13.h, z5.h[3]\n" @@ -1310,13 +1333,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z17.h, z9.h, z4.h[4]\n" "fmla z21.h, z9.h, z5.h[4]\n" @@ -1325,12 +1347,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z17.h, z13.h, z4.h[5]\n" "fmla z21.h, z13.h, z5.h[5]\n" @@ -1339,10 +1361,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" @@ -1352,9 +1375,27 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1h z20.h, p0, [c_ptr1]\n" @@ -1364,7 +1405,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -1387,76 +1428,61 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z20.h, #0\n" + "mov z24.d, z16.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.h, #0\n" + "mov z26.d, z18.d\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.h, #0\n" + "mov z27.d, z19.d\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z24.h, #0\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z25.h, #0\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z26.h, #0\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.h, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1872,10 +1898,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n" "fmla z20.h, z8.h, z5.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.h, z8.h, z6.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" @@ -1968,6 +1997,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" @@ -1992,13 +2022,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z24.h, z8.h, z2.h[0]\n" "fmla z17.h, z9.h, z0.h[0]\n" @@ -2011,12 +2040,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z24.h, z12.h, z2.h[1]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -2029,12 +2058,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z24.h, z8.h, z2.h[2]\n" "fmla z17.h, z9.h, z0.h[2]\n" @@ -2047,12 +2077,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z24.h, z12.h, z2.h[3]\n" "fmla z17.h, z13.h, z0.h[3]\n" @@ -2065,13 +2095,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z24.h, z8.h, z2.h[4]\n" "fmla z17.h, z9.h, z0.h[4]\n" @@ -2084,12 +2113,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z24.h, z12.h, z2.h[5]\n" "fmla z17.h, z13.h, z0.h[5]\n" @@ -2102,10 +2131,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" @@ -2125,18 +2155,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z24.h, z8.h, z2.h[0]\n" - "ld1rqh z5.h, p6/z, [a_ptr1]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z6.h, p6/z, [a_ptr2]\n" + "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "ld1rqh z6.h, p6/z, [a_ptr2]\n" "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -2222,6 +2255,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" @@ -2246,13 +2280,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z24.h, z8.h, z6.h[0]\n" "fmla z17.h, z9.h, z4.h[0]\n" @@ -2265,12 +2298,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z24.h, z12.h, z6.h[1]\n" "fmla z17.h, z13.h, z4.h[1]\n" @@ -2283,12 +2316,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z24.h, z8.h, z6.h[2]\n" "fmla z17.h, z9.h, z4.h[2]\n" @@ -2301,12 +2335,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z24.h, z12.h, z6.h[3]\n" "fmla z17.h, z13.h, z4.h[3]\n" @@ -2319,13 +2353,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z24.h, z8.h, z6.h[4]\n" "fmla z17.h, z9.h, z4.h[4]\n" @@ -2338,12 +2371,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z24.h, z12.h, z6.h[5]\n" "fmla z17.h, z13.h, z4.h[5]\n" @@ -2356,10 +2389,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" @@ -2373,13 +2407,39 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" "st1h z24.h, p0, [c_ptr2]\n" @@ -2391,7 +2451,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -2419,92 +2479,73 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "mov z20.h, #0\n" + "mov z24.d, z16.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.h, #0\n" + "mov z26.d, z18.d\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.h, #0\n" + "mov z27.d, z19.d\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.h, #0\n" + "mov z28.d, z16.d\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z25.h, #0\n" + "mov z29.d, z17.d\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z26.h, #0\n" + "mov z30.d, z18.d\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z27.h, #0\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.h, #0\n" + "mov z31.d, z19.d\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.h, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.h, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.h, #0\n" "add a_ptr3, a_ptr3, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" "ld1h z28.h, p0/z, [c_ptr3]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z28.h, p7/m, z28.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z29.h, p7/m, z29.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z30.h, p7/m, z30.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z31.h, p7/m, z31.h, z15.h\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -3021,11 +3062,15 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n" "fmla z20.h, z8.h, z5.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.h, z8.h, z6.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z28.h, z8.h, z7.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "fmla z29.h, z9.h, z7.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3141,6 +3186,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z31.h, z15.h, z7.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" "fmla z28.h, z8.h, z7.h[6]\n" @@ -3173,13 +3219,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z24.h, z8.h, z2.h[0]\n" "fmla z28.h, z8.h, z3.h[0]\n" @@ -3196,12 +3241,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[0]\n" "fmla z31.h, z11.h, z3.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z24.h, z12.h, z2.h[1]\n" "fmla z28.h, z12.h, z3.h[1]\n" @@ -3218,12 +3263,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[1]\n" "fmla z31.h, z15.h, z3.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z24.h, z8.h, z2.h[2]\n" "fmla z28.h, z8.h, z3.h[2]\n" @@ -3240,12 +3286,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[2]\n" "fmla z31.h, z11.h, z3.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z24.h, z12.h, z2.h[3]\n" "fmla z28.h, z12.h, z3.h[3]\n" @@ -3262,13 +3308,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[3]\n" "fmla z31.h, z15.h, z3.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z24.h, z8.h, z2.h[4]\n" "fmla z28.h, z8.h, z3.h[4]\n" @@ -3285,12 +3330,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[4]\n" "fmla z31.h, z11.h, z3.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z24.h, z12.h, z2.h[5]\n" "fmla z28.h, z12.h, z3.h[5]\n" @@ -3307,10 +3352,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[5]\n" "fmla z31.h, z15.h, z3.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" @@ -3336,17 +3382,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z28.h, z8.h, z3.h[0]\n" - "ld1rqh z6.h, p6/z, [a_ptr2]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z7.h, p6/z, [a_ptr3]\n" + "ld1rqh z6.h, p6/z, [a_ptr2]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "ld1rqh z7.h, p6/z, [a_ptr3]\n" "fmla z25.h, z9.h, z2.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.h, z10.h, z3.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" @@ -3456,6 +3506,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z31.h, z15.h, z3.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" "fmla z28.h, z8.h, z3.h[6]\n" @@ -3488,13 +3539,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[7]\n" "fmla z31.h, z15.h, z3.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z24.h, z8.h, z6.h[0]\n" "fmla z28.h, z8.h, z7.h[0]\n" @@ -3511,12 +3561,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[0]\n" "fmla z31.h, z11.h, z7.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z24.h, z12.h, z6.h[1]\n" "fmla z28.h, z12.h, z7.h[1]\n" @@ -3533,12 +3583,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[1]\n" "fmla z31.h, z15.h, z7.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z24.h, z8.h, z6.h[2]\n" "fmla z28.h, z8.h, z7.h[2]\n" @@ -3555,12 +3606,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[2]\n" "fmla z31.h, z11.h, z7.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z24.h, z12.h, z6.h[3]\n" "fmla z28.h, z12.h, z7.h[3]\n" @@ -3577,13 +3628,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[3]\n" "fmla z31.h, z15.h, z7.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z24.h, z8.h, z6.h[4]\n" "fmla z28.h, z8.h, z7.h[4]\n" @@ -3600,12 +3650,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[4]\n" "fmla z31.h, z11.h, z7.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z24.h, z12.h, z6.h[5]\n" "fmla z28.h, z12.h, z7.h[5]\n" @@ -3622,10 +3672,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[5]\n" "fmla z31.h, z15.h, z7.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" @@ -3643,16 +3694,50 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[6]\n" "fmla z31.h, z11.h, z7.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmax z28.h, p7/m, z28.h, z14.h\n" + "fmax z29.h, p7/m, z29.h, z14.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" + "fmax z30.h, p7/m, z30.h, z14.h\n" + "fmin z28.h, p7/m, z28.h, z15.h\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.h, p7/m, z29.h, z15.h\n" + "fmax z31.h, p7/m, z31.h, z14.h\n" + "fmin z30.h, p7/m, z30.h, z15.h\n" "st1h z24.h, p0, [c_ptr2]\n" + "fmin z31.h, p7/m, z31.h, z15.h\n" "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n" "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n" "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n" @@ -3667,11 +3752,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp index 76f452d963..8e3c17917b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, float, int, int, int); +void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); class hybrid_fp32_mla_4VLx4 { @@ -40,10 +40,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<float>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_fp32_mla_4VLx4; - hybrid_fp32_mla_4VLx4(const CPUInfo *ci) - { - - } + hybrid_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp index 1b2d7852a8..855d27a151 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,37 +25,58 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long leftovers = K; + float nullbias[256]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(float); float *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(float); for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>())); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = leftovers; const float *a_ptr0 = a_ptr0_base; const float *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -69,269 +90,283 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.s, #0\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "b 2f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "2:\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "fmla z17.s, z13.s, z4.s[3]\n" + "fmla z18.s, z14.s, z4.s[3]\n" + "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "fmla z18.s, z14.s, z4.s[3]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" + "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.s, z13.s, z4.s[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z18.s, z14.s, z4.s[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z19.s, z15.s, z4.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z19.s, z11.s, z0.s[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.s, z12.s, z0.s[3]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -350,83 +385,74 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.s, #0\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z23.s, #0\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z20.s, z12.s, z1.s[1]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z21.s, z13.s, z1.s[1]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -440,12 +466,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z1.s[2]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z22.s, z10.s, z1.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" @@ -491,12 +517,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z5.s[2]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z4.s[2]\n" + "fmla z22.s, z10.s, z5.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" @@ -510,19 +536,19 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -545,12 +571,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z1.s[2]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z22.s, z10.s, z1.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" @@ -571,9 +597,11 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.s, z9.s, z5.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z22.s, z10.s, z5.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" @@ -592,6 +620,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" @@ -607,14 +636,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z5.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z17.s, z9.s, z0.s[0]\n" "fmla z21.s, z9.s, z1.s[0]\n" @@ -622,13 +650,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z1.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" @@ -636,11 +664,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z1.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" @@ -649,23 +678,26 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z1.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z21.s, z9.s, z1.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z20.s, z12.s, z1.s[1]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" @@ -678,6 +710,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" @@ -693,14 +726,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z1.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" @@ -708,13 +740,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z5.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" @@ -722,11 +754,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z5.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" @@ -735,10 +768,28 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1w z20.s, p0, [c_ptr1]\n" @@ -748,7 +799,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -760,110 +811,97 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "c_ptr2 .req X3\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" "whilelt p1.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z24.d, z16.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.s, #0\n" + "mov z26.d, z18.d\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.s, #0\n" + "mov z27.d, z19.d\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z24.s, #0\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z25.s, #0\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z26.s, #0\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" @@ -887,8 +925,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z21.s, z9.s, z1.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -955,8 +993,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z21.s, z9.s, z5.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" @@ -982,19 +1020,19 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" @@ -1026,8 +1064,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z21.s, z9.s, z1.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -1059,10 +1097,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.s, z8.s, z6.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" @@ -1090,6 +1131,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z27.s, z15.s, z6.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" @@ -1113,14 +1155,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z24.s, z8.s, z2.s[0]\n" "fmla z17.s, z9.s, z0.s[0]\n" @@ -1132,13 +1173,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z17.s, z13.s, z0.s[1]\n" @@ -1150,11 +1191,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" @@ -1167,24 +1209,28 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1rqw z6.s, p6/z, [a_ptr2]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1205,6 +1251,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z27.s, z15.s, z2.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" @@ -1228,14 +1275,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z17.s, z9.s, z4.s[0]\n" @@ -1247,13 +1293,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z17.s, z13.s, z4.s[1]\n" @@ -1265,11 +1311,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" @@ -1282,14 +1329,40 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" "st1w z24.s, p0, [c_ptr2]\n" @@ -1301,7 +1374,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1316,132 +1389,115 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "c_ptr3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" "whilelt p1.s, %[temp], %[width]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" "incw %[temp], all, mul #1\n" "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "mov z20.s, #0\n" + "mov z24.d, z16.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.s, #0\n" + "mov z26.d, z18.d\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.s, #0\n" + "mov z27.d, z19.d\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.s, #0\n" + "mov z28.d, z16.d\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z25.s, #0\n" + "mov z29.d, z17.d\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z26.s, #0\n" + "mov z30.d, z18.d\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z27.s, #0\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "mov z28.s, #0\n" + "mov z31.d, z19.d\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z28.s, p7/m, z28.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z29.s, p7/m, z29.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p7/z, [a_ptr3]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "add a_ptr3, a_ptr3, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" @@ -1471,8 +1527,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1556,8 +1612,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "fmla z29.s, z9.s, z7.s[2]\n" @@ -1591,21 +1647,21 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p7/z, [a_ptr3]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1644,8 +1700,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1686,11 +1742,15 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.s, z8.s, z6.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z28.s, z8.s, z7.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1725,6 +1785,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z31.s, z15.s, z7.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" @@ -1756,14 +1817,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z24.s, z8.s, z2.s[0]\n" "fmla z28.s, z8.s, z3.s[0]\n" @@ -1779,13 +1839,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" @@ -1801,11 +1861,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" "fmla z31.s, z15.s, z3.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" @@ -1822,25 +1883,30 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" "fmla z31.s, z11.s, z3.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p6/z, [a_ptr2]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p6/z, [a_ptr3]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z6.s, p6/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1rqw z7.s, p6/z, [a_ptr3]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1869,6 +1935,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z31.s, z15.s, z3.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" @@ -1900,14 +1967,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" "fmla z31.s, z15.s, z3.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z28.s, z8.s, z7.s[0]\n" @@ -1923,13 +1989,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" "fmla z31.s, z11.s, z7.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" @@ -1945,11 +2011,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" "fmla z31.s, z15.s, z7.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" @@ -1966,17 +2033,51 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" "fmla z31.s, z11.s, z7.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmax z28.s, p7/m, z28.s, z14.s\n" + "fmax z29.s, p7/m, z29.s, z14.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" + "fmax z30.s, p7/m, z30.s, z14.s\n" + "fmin z28.s, p7/m, z28.s, z15.s\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.s, p7/m, z29.s, z15.s\n" + "fmax z31.s, p7/m, z31.s, z14.s\n" + "fmin z30.s, p7/m, z30.s, z15.s\n" "st1w z24.s, p0, [c_ptr2]\n" + "fmin z31.s, p7/m, z31.s, z15.s\n" "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n" "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n" "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n" @@ -1991,11 +2092,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp index ffd7918b7a..d8422105cc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class hybrid_s8s32_dot_4VLx4 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<int32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_s8s32_dot_4VLx4; - hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) - { - - } + hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp index 673f186524..d57557a242 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 const unsigned long ldab = lda * sizeof(int8_t); int32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(int32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>())); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const int8_t *a_ptr0 = a_ptr0_base; const int8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(int32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "sdot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z18.s, z14.b, z4.b[1]\n" + "sdot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "sdot z19.s, z15.b, z4.b[1]\n" + "sdot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "sdot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "sdot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "sdot z16.s, z12.b, z0.b[1]\n" + "sdot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "sdot z17.s, z13.b, z0.b[1]\n" + "sdot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "sdot z18.s, z14.b, z0.b[1]\n" + "sdot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "sdot z19.s, z15.b, z0.b[1]\n" + "sdot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "sdot z16.s, z8.b, z0.b[2]\n" + "sdot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[2]\n" + "sdot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "sdot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z17.s, z9.b, z0.b[0]\n" "sdot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z17.s, z13.b, z0.b[1]\n" "sdot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "sdot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z17.s, z13.b, z4.b[1]\n" "sdot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "sdot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z24.s, z8.b, z2.b[0]\n" "sdot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z24.s, z12.b, z2.b[1]\n" "sdot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z24.s, z8.b, z6.b[0]\n" "sdot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z24.s, z12.b, z6.b[1]\n" "sdot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "sdot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "sdot z25.s, z9.b, z6.b[0]\n" "sdot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z24.s, z8.b, z2.b[0]\n" "sdot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z24.s, z12.b, z2.b[1]\n" "sdot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "sdot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z24.s, z8.b, z6.b[0]\n" "sdot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z24.s, z12.b, z6.b[1]\n" "sdot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp index 2e51349d35..5dab1da135 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class hybrid_u8u32_dot_4VLx4 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,20 +53,32 @@ public: return get_vector_length<uint32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_u8u32_dot_4VLx4; - hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) - { - - } + hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp index d34d0e5fc7..bf3e8ca26a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const uint8_t *a_ptr0 = a_ptr0_base; const uint8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "udot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z14.b, z4.b[1]\n" + "udot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z19.s, z15.b, z4.b[1]\n" + "udot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "udot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "udot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "udot z17.s, z13.b, z0.b[1]\n" + "udot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "udot z18.s, z14.b, z0.b[1]\n" + "udot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "udot z19.s, z15.b, z0.b[1]\n" + "udot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "udot z16.s, z8.b, z0.b[2]\n" + "udot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "udot z17.s, z9.b, z0.b[2]\n" + "udot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "udot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "udot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp index 2ca4ce25e8..a74f9efb87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_fp16_mla_3VLx8; - interleaved_fp16_mla_3VLx8(const CPUInfo *ci) - { - - } + interleaved_fp16_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp index 8c1fe6d0b6..1e73d8e185 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_fp32_mla_3VLx8; - interleaved_fp32_mla_3VLx8(const CPUInfo *ci) - { - - } + interleaved_fp32_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp index cbb21387b1..8668ea7fbc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_s8s32_dot_3VLx8; - interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) - { - - } + interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp index 99c039e121..d3c8851154 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Arm Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_u8u32_dot_3VLx8; - interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) - { - - } + interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp index 6cce601dcc..741f200d25 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,11 @@ #ifdef __ARM_FEATURE_SVE - - - namespace arm_gemm { // Actual kernel implementations -void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int); +void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); class native_fp16_mla_4VLx4 { @@ -40,10 +37,10 @@ public: typedef __fp16 operand_type; typedef __fp16 result_type; - typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int); + typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +50,32 @@ public: return get_vector_length<__fp16>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + // Default to the generic kernel kern_type kernel=sve_native_fp16_mla_4VLx4; - native_fp16_mla_4VLx4(const CPUInfo *ci) - { - - } + native_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp index f1aaeb13ee..14dd38bd25 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,30 +25,49 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) { const long loops_count = ((K + 8) / 16) - 1; K -= loops_count * 16; const long regs_count = (K / 8) - 1; K -= (regs_count + 1) * 8; const long leftovers = K; + __fp16 nullbias[512]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity()); + const __fp16 * const minptr = &minval; + const __fp16 * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const __fp16 * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(__fp16); __fp16 *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(__fp16); for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>())); - const __fp16 *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -56,6 +75,8 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld const __fp16 *a_ptr0 = a_ptr0_base; const __fp16 *b_ptr0 = B + x0; long ldbb = ldb * sizeof(__fp16); + const unsigned long ldcb = ldc * sizeof(__fp16); + const __fp16 *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -64,52 +85,34 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z18.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "whilelt p2.h, %[temp], %[width]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" + "inch %[temp], all, mul #1\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "cbz %[loops], 2f\n" - "3:\n" + "whilelt p3.h, %[temp], %[width]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" + "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" "fmla z18.h, z10.h, z0.h[0]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "subs %[loops], %[loops], #0x1\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -119,70 +122,69 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" - "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[3]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[3]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[3]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[4]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z9.h, z0.h[4]\n" + "fmla z16.h, z8.h, z0.h[4]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[4]\n" + "fmla z17.h, z9.h, z0.h[4]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[4]\n" + "fmla z18.h, z10.h, z0.h[4]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[4]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[5]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[5]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[5]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[6]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z9.h, z0.h[6]\n" + "fmla z16.h, z8.h, z0.h[6]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[6]\n" + "fmla z17.h, z9.h, z0.h[6]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[6]\n" + "fmla z18.h, z10.h, z0.h[6]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[6]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[7]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[7]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[7]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[7]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.h, z9.h, z4.h[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z4.h[0]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z4.h[0]\n" + "fmla z18.h, z10.h, z4.h[0]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.h, z11.h, z4.h[0]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" @@ -247,94 +249,95 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z18.h, z14.h, z4.h[7]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z15.h, z4.h[7]\n" + "b.ne 2b\n" + "1:\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "b.ne 3b\n" - "2:\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[3]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[3]\n" + "fmla z16.h, z12.h, z0.h[3]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[4]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[4]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[4]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[4]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[5]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[5]\n" + "fmla z16.h, z12.h, z0.h[5]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[5]\n" + "fmla z17.h, z13.h, z0.h[5]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[5]\n" + "fmla z18.h, z14.h, z0.h[5]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[5]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[6]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[6]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[6]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[6]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[7]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[7]\n" + "fmla z16.h, z12.h, z0.h[7]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[7]\n" + "fmla z17.h, z13.h, z0.h[7]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[7]\n" + "fmla z18.h, z14.h, z0.h[7]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[7]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z4.h[0]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.h, z9.h, z4.h[0]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.h, z10.h, z4.h[0]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z19.h, z11.h, z4.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" + "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z13.h, z4.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.h, z14.h, z4.h[1]\n" @@ -392,7 +395,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[7]\n" "fmla z18.h, z14.h, z4.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -403,7 +406,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[0]\n" "fmla z18.h, z10.h, z0.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -414,7 +417,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[1]\n" "fmla z18.h, z14.h, z0.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -425,7 +428,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[2]\n" "fmla z18.h, z10.h, z0.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -436,7 +439,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[3]\n" "fmla z18.h, z14.h, z0.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -447,7 +450,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[4]\n" "fmla z18.h, z10.h, z0.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -458,7 +461,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[5]\n" "fmla z18.h, z14.h, z0.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -468,73 +471,73 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[6]\n" "fmla z18.h, z10.h, z0.h[6]\n" "fmla z19.h, z11.h, z0.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[3]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[3]\n" + "fmla z16.h, z12.h, z0.h[3]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[4]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[4]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[4]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[4]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[5]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[5]\n" + "fmla z16.h, z12.h, z0.h[5]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[5]\n" + "fmla z17.h, z13.h, z0.h[5]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[5]\n" + "fmla z18.h, z14.h, z0.h[5]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[5]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[6]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[6]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[6]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[6]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.h, z12.h, z0.h[7]\n" "fmla z17.h, z13.h, z0.h[7]\n" "fmla z18.h, z14.h, z0.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -545,7 +548,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[0]\n" "fmla z18.h, z10.h, z4.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -556,7 +559,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[1]\n" "fmla z18.h, z14.h, z4.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -567,7 +570,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[2]\n" "fmla z18.h, z10.h, z4.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -578,7 +581,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[3]\n" "fmla z18.h, z14.h, z4.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -589,7 +592,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[4]\n" "fmla z18.h, z10.h, z4.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -600,7 +603,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[5]\n" "fmla z18.h, z14.h, z4.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -610,14 +613,24 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[6]\n" "fmla z18.h, z10.h, z4.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -631,66 +644,33 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z19.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.h, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z23.h, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "inch %[temp], all, mul #1\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z22.d, z18.d\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -906,23 +886,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z22.h, z10.h, z1.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" @@ -1026,9 +1006,11 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z21.h, z9.h, z5.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z23.h, z11.h, z5.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" @@ -1112,7 +1094,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1127,7 +1109,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1142,7 +1124,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1157,7 +1139,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1172,7 +1154,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1187,7 +1169,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1202,7 +1184,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1216,22 +1198,22 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[6]\n" "fmla z19.h, z11.h, z0.h[6]\n" "fmla z23.h, z11.h, z1.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z22.h, z10.h, z1.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" @@ -1239,6 +1221,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z20.h, z12.h, z1.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z13.h, z0.h[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z21.h, z13.h, z1.h[1]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z14.h, z0.h[1]\n" @@ -1315,7 +1298,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1330,7 +1313,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1345,7 +1328,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1360,7 +1343,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1375,7 +1358,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1390,7 +1373,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1405,7 +1388,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1419,10 +1402,28 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1h z20.h, p0, [c_ptr1]\n" @@ -1432,7 +1433,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -1450,82 +1451,39 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" - "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z20.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.h, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z24.h, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "mov z25.h, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z26.h, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z27.h, #0\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" - "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" - "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "inch %[temp], all, mul #1\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" + "mov z26.d, z18.d\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -1551,9 +1509,9 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z20.h, z12.h, z1.h[1]\n" "add a_ptr2, a_ptr2, #0x20\n" + "fmla z20.h, z12.h, z1.h[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.h, z12.h, z2.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -1808,23 +1766,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p7/z, [a_ptr2]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "fmla z22.h, z10.h, z1.h[0]\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -1960,10 +1918,13 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z24.h, z8.h, z6.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "fmla z26.h, z10.h, z6.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -2080,7 +2041,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2099,7 +2060,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2118,7 +2079,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2137,7 +2098,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2156,7 +2117,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2175,7 +2136,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2194,7 +2155,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2212,26 +2173,28 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[6]\n" "fmla z23.h, z11.h, z1.h[6]\n" "fmla z27.h, z11.h, z2.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p6/z, [a_ptr2]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" @@ -2344,7 +2307,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2363,7 +2326,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2382,7 +2345,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2401,7 +2364,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2420,7 +2383,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2439,7 +2402,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2458,7 +2421,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2476,14 +2439,40 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" "st1h z24.h, p0, [c_ptr2]\n" @@ -2495,7 +2484,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -2518,98 +2507,45 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z28.d, z16.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "mov z20.h, #0\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.h, #0\n" + "mov z29.d, z17.d\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.h, #0\n" + "inch %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z25.h, #0\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z26.h, #0\n" + "mov z26.d, z18.d\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z30.d, z18.d\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z27.h, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "mov z28.h, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z29.h, #0\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "mov z30.h, #0\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z31.h, #0\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" - "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" - "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" - "ld1h z28.h, p0/z, [c_ptr3]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" - "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" - "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" - "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z28.h, p7/m, z28.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z29.h, p7/m, z29.h, z15.h\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z30.h, p7/m, z30.h, z15.h\n" - "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z31.h, p7/m, z31.h, z15.h\n" - "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "mov z31.d, z19.d\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -2959,23 +2895,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z28.h, z8.h, z3.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p7/z, [a_ptr2]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z28.h, z8.h, z3.h[0]\n" "ld1rqh z7.h, p7/z, [a_ptr3]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" @@ -3143,11 +3079,15 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z20.h, z8.h, z5.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.h, z8.h, z6.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z28.h, z8.h, z7.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z29.h, z9.h, z7.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" @@ -3297,7 +3237,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3320,7 +3260,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "fmla z31.h, z11.h, z3.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3343,7 +3283,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" "fmla z31.h, z15.h, z3.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3366,7 +3306,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" "fmla z31.h, z11.h, z3.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3389,7 +3329,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" "fmla z31.h, z15.h, z3.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3412,7 +3352,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" "fmla z31.h, z11.h, z3.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3435,7 +3375,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" "fmla z31.h, z15.h, z3.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3457,27 +3397,30 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[6]\n" "fmla z27.h, z11.h, z2.h[6]\n" "fmla z31.h, z11.h, z3.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z28.h, z8.h, z3.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p6/z, [a_ptr2]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z28.h, z8.h, z3.h[0]\n" "ld1rqh z7.h, p6/z, [a_ptr3]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "fmla z25.h, z9.h, z2.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.h, z10.h, z3.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" @@ -3622,7 +3565,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" "fmla z31.h, z15.h, z3.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3645,7 +3588,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" "fmla z31.h, z11.h, z7.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3668,7 +3611,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" "fmla z31.h, z15.h, z7.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3691,7 +3634,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" "fmla z31.h, z11.h, z7.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3714,7 +3657,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" "fmla z31.h, z15.h, z7.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3737,7 +3680,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" "fmla z31.h, z11.h, z7.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3760,7 +3703,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" "fmla z31.h, z15.h, z7.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3782,17 +3725,51 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" "fmla z31.h, z11.h, z7.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmax z28.h, p7/m, z28.h, z14.h\n" + "fmax z29.h, p7/m, z29.h, z14.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" + "fmax z30.h, p7/m, z30.h, z14.h\n" + "fmin z28.h, p7/m, z28.h, z15.h\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.h, p7/m, z29.h, z15.h\n" + "fmax z31.h, p7/m, z31.h, z14.h\n" + "fmin z30.h, p7/m, z30.h, z15.h\n" "st1h z24.h, p0, [c_ptr2]\n" + "fmin z31.h, p7/m, z31.h, z15.h\n" "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n" "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n" "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n" @@ -3807,11 +3784,12 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp index d7f9f20074..19e5fbd974 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,13 +25,11 @@ #ifdef __ARM_FEATURE_SVE - - namespace arm_gemm { // Actual kernel implementations -void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int); +void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); class native_fp32_mla_4VLx4 { @@ -39,10 +37,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -52,20 +50,32 @@ public: return get_vector_length<float>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + // Default to the generic kernel kern_type kernel=sve_native_fp32_mla_4VLx4; - native_fp32_mla_4VLx4(const CPUInfo *ci) - { - - } + native_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp index 6e225669fc..3fc0e5fa36 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,30 +25,49 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long leftovers = K; + float nullbias[256]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(float); float *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(float); for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>())); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -56,6 +75,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, const float *a_ptr0 = a_ptr0_base; const float *b_ptr0 = B + x0; long ldbb = ldb * sizeof(float); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -64,191 +85,173 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "whilelt p2.s, %[temp], %[width]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" + "incw %[temp], all, mul #1\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "cbz %[loops], 1f\n" "2:\n" - "cbz %[loops], 3f\n" - "4:\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z16.s, z12.s, z0.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z19.s, z15.s, z0.s[1]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "fmla z18.s, z14.s, z0.s[3]\n" - "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z14.s, z0.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "fmla z18.s, z10.s, z4.s[0]\n" - "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" - "fmla z18.s, z14.s, z4.s[1]\n" - "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z4.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" - "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z4.s[3]\n" - "fmla z17.s, z13.s, z4.s[3]\n" - "fmla z18.s, z14.s, z4.s[3]\n" - "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z4.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z4.s[3]\n" + "b.ne 2b\n" + "1:\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "b.ne 4b\n" - "3:\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z18.s, z10.s, z0.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.s, z11.s, z0.s[0]\n" - "fmla z16.s, z12.s, z0.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" - "fmla z18.s, z14.s, z0.s[1]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z15.s, z0.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z19.s, z15.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" - "fmla z18.s, z14.s, z0.s[3]\n" - "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" - "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" "fmla z17.s, z13.s, z4.s[3]\n" "fmla z18.s, z14.s, z4.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -259,7 +262,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -270,7 +273,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z13.s, z0.s[1]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -280,37 +283,37 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z18.s, z10.s, z0.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.s, z11.s, z0.s[0]\n" - "fmla z16.s, z12.s, z0.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" - "fmla z18.s, z14.s, z0.s[1]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z15.s, z0.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z19.s, z15.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.s, z12.s, z0.s[3]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -321,7 +324,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z4.s[0]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -332,7 +335,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z13.s, z4.s[1]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -342,14 +345,24 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z4.s[2]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -363,61 +376,33 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z23.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" + "incw %[temp], all, mul #1\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z22.d, z18.d\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "2:\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "cbz %[loops], 3f\n" - "4:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -425,200 +410,202 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z23.s, z11.s, z1.s[0]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z16.s, z12.s, z0.s[1]\n" + "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z20.s, z12.s, z1.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z13.s, z0.s[1]\n" + "fmla z20.s, z12.s, z1.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z17.s, z13.s, z0.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z22.s, z14.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[2]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z20.s, z8.s, z1.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z12.s, z0.s[3]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z12.s, z1.s[3]\n" + "fmla z16.s, z12.s, z0.s[3]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z20.s, z12.s, z1.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z22.s, z14.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z8.s, z4.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z16.s, z8.s, z4.s[0]\n" + "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z22.s, z10.s, z5.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z22.s, z14.s, z5.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z5.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z22.s, z10.s, z5.s[2]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[3]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[3]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z5.s[3]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[3]\n" "fmla z22.s, z14.s, z5.s[3]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "b.ne 4b\n" - "3:\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z22.s, z14.s, z1.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z22.s, z14.s, z1.s[3]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z22.s, z10.s, z5.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z23.s, z11.s, z5.s[0]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z22.s, z14.s, z5.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z20.s, z8.s, z5.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" @@ -631,7 +618,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z5.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -646,7 +633,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z1.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -661,7 +648,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z1.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -675,40 +662,41 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z1.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z21.s, z13.s, z1.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z22.s, z14.s, z1.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" @@ -722,7 +710,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z1.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -737,7 +725,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z5.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -752,7 +740,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z5.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -766,10 +754,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1w z20.s, p0, [c_ptr1]\n" @@ -779,7 +785,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -791,85 +797,45 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "c_ptr2 .req X3\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.s, #0\n" + "incw %[temp], all, mul #1\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z24.s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "mov z25.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z26.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z27.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" + "mov z26.d, z18.d\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "2:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -879,28 +845,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z26.s, z10.s, z2.s[0]\n" + "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[0]\n" + "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z20.s, z12.s, z1.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z12.s, z2.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -916,9 +882,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -933,9 +899,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" @@ -953,12 +919,12 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" "fmla z25.s, z9.s, z6.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[0]\n" "fmla z22.s, z10.s, z5.s[0]\n" "fmla z26.s, z10.s, z6.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -970,9 +936,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" @@ -987,9 +953,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z6.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" @@ -1004,9 +970,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[3]\n" "fmla z24.s, z12.s, z6.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[3]\n" "fmla z21.s, z13.s, z5.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[3]\n" @@ -1016,37 +982,37 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z22.s, z10.s, z1.s[0]\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -1061,9 +1027,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -1078,9 +1044,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" @@ -1098,12 +1064,15 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z22.s, z10.s, z5.s[0]\n" "fmla z26.s, z10.s, z6.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -1115,9 +1084,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" @@ -1152,7 +1121,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1171,7 +1140,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1190,7 +1159,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1208,36 +1177,38 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -1272,7 +1243,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1291,7 +1262,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1310,7 +1281,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1328,14 +1299,40 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" "st1w z24.s, p0, [c_ptr2]\n" @@ -1347,7 +1344,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1362,103 +1359,53 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "c_ptr3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "ptrue p7.s\n" - "whilelt p1.s, %[temp], %[width]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" + "whilelt p6.s, %[temp], %[leftovers]\n" + "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[temp], %[width]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z28.d, z16.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "mov z20.s, #0\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.s, #0\n" + "mov z29.d, z17.d\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.s, #0\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z25.s, #0\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z26.s, #0\n" + "mov z26.d, z18.d\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z30.d, z18.d\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z27.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "mov z28.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z28.s, p7/m, z28.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z29.s, p7/m, z29.s, z15.s\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z30.s, p7/m, z30.s, z15.s\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmul z31.s, p7/m, z31.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "2:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "mov z31.d, z19.d\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -1470,23 +1417,23 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z29.s, z9.s, z3.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z22.s, z10.s, z1.s[0]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z22.s, z10.s, z1.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr3, a_ptr3, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1495,8 +1442,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1516,8 +1463,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1537,8 +1484,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" "fmla z28.s, z12.s, z3.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" "fmla z25.s, z13.s, z2.s[3]\n" "fmla z29.s, z13.s, z3.s[3]\n" @@ -1562,9 +1509,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z28.s, z8.s, z7.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z6.s[0]\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1583,8 +1530,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" "fmla z25.s, z13.s, z6.s[1]\n" "fmla z29.s, z13.s, z7.s[1]\n" @@ -1604,8 +1551,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "fmla z29.s, z9.s, z7.s[2]\n" @@ -1625,8 +1572,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[3]\n" "fmla z24.s, z12.s, z6.s[3]\n" "fmla z28.s, z12.s, z7.s[3]\n" - "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[3]\n" "fmla z21.s, z13.s, z5.s[3]\n" "fmla z25.s, z13.s, z6.s[3]\n" "fmla z29.s, z13.s, z7.s[3]\n" @@ -1640,28 +1587,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z28.s, z8.s, z3.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z28.s, z8.s, z3.s[0]\n" "ld1rqw z7.s, p7/z, [a_ptr3]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "fmla z29.s, z9.s, z3.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z22.s, z10.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1674,8 +1621,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1695,8 +1642,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1716,8 +1663,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" "fmla z28.s, z12.s, z3.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" "fmla z25.s, z13.s, z2.s[3]\n" "fmla z29.s, z13.s, z3.s[3]\n" @@ -1740,11 +1687,15 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z28.s, z8.s, z7.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" @@ -1762,8 +1713,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" "fmla z25.s, z13.s, z6.s[1]\n" "fmla z29.s, z13.s, z7.s[1]\n" @@ -1810,7 +1761,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1833,7 +1784,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1856,7 +1807,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" "fmla z31.s, z15.s, z3.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1878,27 +1829,30 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" "fmla z31.s, z11.s, z3.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z28.s, z8.s, z3.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z28.s, z8.s, z3.s[0]\n" "ld1rqw z7.s, p6/z, [a_ptr3]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.s, z9.s, z3.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.s, z10.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1911,8 +1865,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1959,7 +1913,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" "fmla z31.s, z15.s, z3.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1982,7 +1936,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" "fmla z31.s, z11.s, z7.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -2005,7 +1959,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" "fmla z31.s, z15.s, z7.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2027,17 +1981,51 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" "fmla z31.s, z11.s, z7.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmax z28.s, p7/m, z28.s, z14.s\n" + "fmax z29.s, p7/m, z29.s, z14.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" + "fmax z30.s, p7/m, z30.s, z14.s\n" + "fmin z28.s, p7/m, z28.s, z15.s\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.s, p7/m, z29.s, z15.s\n" + "fmax z31.s, p7/m, z31.s, z14.s\n" + "fmin z30.s, p7/m, z30.s, z15.s\n" "st1w z24.s, p0, [c_ptr2]\n" + "fmin z31.s, p7/m, z31.s, z15.s\n" "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n" "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n" "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n" @@ -2052,11 +2040,12 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp index 8b98358cd4..1b9d1312b5 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,11 +27,12 @@ #include <cstdint> + namespace arm_gemm { // Actual kernel implementations -void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int); +void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class native_s8s32_dot_4VLx4 { @@ -39,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -52,20 +53,32 @@ public: return get_vector_length<int32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + // Default to the generic kernel kern_type kernel=sve_native_s8s32_dot_4VLx4; - native_s8s32_dot_4VLx4(const CPUInfo *ci) - { - - } + native_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp index abee1bbe1f..95cf88ab54 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; const long regs_count = (K / 16) - 1; @@ -46,11 +46,9 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l const unsigned long ldab = lda * sizeof(int8_t); int32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(int32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>())); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -62,95 +60,68 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l const int8_t *b_ptr2 = b_ptr1 + ldb; const int8_t *b_ptr3 = b_ptr2 + ldb; long ldbb = ldb * sizeof(int8_t) * 4; + const unsigned long ldcb = ldc * sizeof(int32_t); switch(M-y) { case 1: __asm __volatile ( + "mov z16.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z17.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z18.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z19.s, #0\n" "incw %[temp], all, mul #1\n" "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z19.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "whilelt p1.s, %[temp], %[width]\n" + "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip2 z11.b, z8.b, z9.b\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" + "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "whilelt p3.s, %[temp], %[width]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z10.b, z11.b, z12.b\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "sdot z16.s, z8.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "zip2 z15.b, z12.b, z13.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "zip2 z15.b, z12.b, z13.b\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z12.b, z13.b, z14.b\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" @@ -289,32 +260,32 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "b.ne 3b\n" - "2:\n" "zip1 z10.b, z10.b, z8.b\n" "zip1 z8.b, z9.b, z10.b\n" + "b.ne 2b\n" + "1:\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" @@ -377,6 +348,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z10.b, z10.b, z8.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z8.b, z9.b, z10.b\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" @@ -437,7 +409,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -459,7 +431,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -481,7 +453,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -502,31 +474,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -540,33 +512,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -580,33 +552,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -620,33 +592,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -660,36 +632,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z16.s, z12.b, z0.b[1]\n" + "sdot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z13.b, z0.b[1]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "sdot z16.s, z12.b, z0.b[1]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "sdot z17.s, z13.b, z0.b[1]\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z14.b, z0.b[1]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z15.b, z0.b[1]\n" @@ -718,7 +690,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -740,7 +712,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -762,7 +734,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -783,31 +755,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -821,33 +793,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -861,33 +833,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -901,33 +873,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -941,14 +913,14 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -956,125 +928,81 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" + "mov z17.s, #0\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z18.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z19.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z20.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z21.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z20.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z21.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p1.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ptrue p7.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z22.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z23.s, #0\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "ld1rqb z1.b, p7/z, [a_ptr1]\n" "zip1 z10.b, z10.b, z8.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "zip1 z10.b, z11.b, z12.b\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" "subs %[loops], %[loops], #0x1\n" - "sdot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -1244,38 +1172,38 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "sdot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1357,23 +1285,25 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z20.s, z8.b, z5.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z14.b, z14.b, z12.b\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z4.b[0]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z18.s, z10.b, z4.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z23.s, z11.b, z5.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "sdot z23.s, z11.b, z5.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1420,7 +1350,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1446,7 +1376,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1472,7 +1402,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1497,31 +1427,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1539,33 +1469,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1583,33 +1513,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1627,33 +1557,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1671,35 +1601,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "sdot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1746,7 +1677,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1772,7 +1703,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1798,7 +1729,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1823,31 +1754,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1865,33 +1796,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1909,33 +1840,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1953,33 +1884,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1997,7 +1928,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -2010,7 +1941,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -2020,119 +1951,65 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z17.s, #0\n" "add a_ptr2, a_ptr1, %[lda]\n" + "mov z18.s, #0\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z19.s, #0\n" "add c_ptr2, c_ptr1, %[ldc]\n" + "mov z20.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z21.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z22.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z23.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z24.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z21.s, #0\n" + "mov z26.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z22.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "incw %[temp], all, mul #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z23.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z24.s, #0\n" + "ptrue p7.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "whilelt p2.s, %[temp], %[width]\n" "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "mov z25.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "mov z26.s, #0\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "mov z27.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z8.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z9.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "zip2 z11.b, z11.b, z12.b\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "sdot z24.s, z8.b, z2.b[0]\n" @@ -2361,24 +2238,23 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" @@ -2386,12 +2262,13 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -2491,8 +2368,11 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip1 z14.b, z14.b, z12.b\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -2571,7 +2451,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2601,7 +2481,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -2631,7 +2511,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2660,31 +2540,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2706,33 +2586,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2754,33 +2634,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2802,33 +2682,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2850,34 +2730,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -2942,7 +2824,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2972,7 +2854,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3002,7 +2884,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3031,31 +2913,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3077,33 +2959,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3125,33 +3007,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3173,33 +3055,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3221,7 +3103,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -3240,7 +3122,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -3253,175 +3135,109 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "whilelt p6.b, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "whilelt p4.b, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" + "add a_ptr3, a_ptr2, %[lda]\n" "mov z19.s, #0\n" - "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "mov z20.s, #0\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "mov z21.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "mov z22.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "whilelt p6.b, %[temp], %[leftovers]\n" "mov z23.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "whilelt p0.s, %[temp], %[width]\n" "mov z24.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "whilelt p4.b, %[temp], %[width]\n" "mov z25.s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" - "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "incw %[temp], all, mul #1\n" "mov z26.s, #0\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "mov z27.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip2 z9.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "mov z28.s, #0\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "mov z29.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" + "zip2 z11.b, z8.b, z9.b\n" + "incw %[temp], all, mul #1\n" + "zip1 z9.b, z8.b, z9.b\n" + "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "mov z30.s, #0\n" - "zip2 z15.b, z12.b, z13.b\n" - "zip1 z13.b, z12.b, z13.b\n" + "ptrue p7.b\n" "mov z31.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "zip2 z12.b, z10.b, z8.b\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "incw %[temp], all, mul #1\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "zip1 z8.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip2 z9.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z11.b, z11.b, z12.b\n" "add a_ptr3, a_ptr3, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1rqb z6.b, p7/z, [a_ptr2]\n" "sdot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z7.b, p7/z, [a_ptr3]\n" "sdot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p7/z, [a_ptr2]\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z8.b, z14.b, z12.b\n" - "ld1rqb z7.b, p7/z, [a_ptr3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "add a_ptr3, a_ptr3, #0x20\n" "sdot z26.s, z10.b, z2.b[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -3649,29 +3465,29 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z28.s, z8.b, z3.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "sdot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p7/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3797,9 +3613,13 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z28.s, z8.b, z7.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr3, a_ptr3, #2\n" "zip1 z14.b, z14.b, z12.b\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -3893,7 +3713,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3927,7 +3747,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3961,7 +3781,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3994,31 +3814,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4044,33 +3864,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4096,33 +3916,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4148,33 +3968,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4200,37 +4020,40 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z28.s, z8.b, z3.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "sdot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p6/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" @@ -4309,7 +4132,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4343,7 +4166,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -4377,7 +4200,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4410,31 +4233,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4460,33 +4283,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4512,33 +4335,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4564,33 +4387,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4616,7 +4439,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -4641,11 +4464,12 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp index bcbd3d35f5..33e3ac6c23 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int); +void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class native_u8u32_dot_4VLx4 { @@ -40,10 +40,10 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<uint32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + // Default to the generic kernel kern_type kernel=sve_native_u8u32_dot_4VLx4; - native_u8u32_dot_4VLx4(const CPUInfo *ci) - { - - } + native_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp index cdcea59c5e..994d608bf6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; const long regs_count = (K / 16) - 1; @@ -46,11 +46,9 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -62,95 +60,68 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int const uint8_t *b_ptr2 = b_ptr1 + ldb; const uint8_t *b_ptr3 = b_ptr2 + ldb; long ldbb = ldb * sizeof(uint8_t) * 4; + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: __asm __volatile ( + "mov z16.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z17.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z18.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z19.s, #0\n" "incw %[temp], all, mul #1\n" "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z19.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "whilelt p1.s, %[temp], %[width]\n" + "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip2 z11.b, z8.b, z9.b\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" + "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "whilelt p3.s, %[temp], %[width]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z10.b, z11.b, z12.b\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "udot z16.s, z8.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "zip2 z15.b, z12.b, z13.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "zip2 z15.b, z12.b, z13.b\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "udot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z12.b, z13.b, z14.b\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" @@ -289,32 +260,32 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "b.ne 3b\n" - "2:\n" "zip1 z10.b, z10.b, z8.b\n" "zip1 z8.b, z9.b, z10.b\n" + "b.ne 2b\n" + "1:\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" @@ -377,6 +348,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z10.b, z10.b, z8.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z8.b, z9.b, z10.b\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" @@ -437,7 +409,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -459,7 +431,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -481,7 +453,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -502,31 +474,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -540,33 +512,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -580,33 +552,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -620,33 +592,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -660,36 +632,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z17.s, z9.b, z0.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z13.b, z0.b[1]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "udot z16.s, z12.b, z0.b[1]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "udot z17.s, z13.b, z0.b[1]\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z14.b, z0.b[1]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z15.b, z0.b[1]\n" @@ -718,7 +690,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -740,7 +712,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -762,7 +734,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -783,31 +755,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -821,33 +793,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -861,33 +833,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -901,33 +873,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -941,14 +913,14 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -956,125 +928,81 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" + "mov z17.s, #0\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z18.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z19.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z20.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z21.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z20.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z21.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p1.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ptrue p7.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z22.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z23.s, #0\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "ld1rqb z1.b, p7/z, [a_ptr1]\n" "zip1 z10.b, z10.b, z8.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "zip1 z10.b, z11.b, z12.b\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" "subs %[loops], %[loops], #0x1\n" - "udot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -1244,38 +1172,38 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "udot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1357,23 +1285,25 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z20.s, z8.b, z5.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z14.b, z14.b, z12.b\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z4.b[0]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z18.s, z10.b, z4.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z4.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z23.s, z11.b, z5.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "udot z23.s, z11.b, z5.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1420,7 +1350,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1446,7 +1376,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1472,7 +1402,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1497,31 +1427,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1539,33 +1469,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1583,33 +1513,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1627,33 +1557,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1671,35 +1601,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "udot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1746,7 +1677,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1772,7 +1703,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1798,7 +1729,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1823,31 +1754,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1865,33 +1796,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1909,33 +1840,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1953,33 +1884,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1997,7 +1928,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -2010,7 +1941,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -2020,119 +1951,65 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z17.s, #0\n" "add a_ptr2, a_ptr1, %[lda]\n" + "mov z18.s, #0\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z19.s, #0\n" "add c_ptr2, c_ptr1, %[ldc]\n" + "mov z20.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z21.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z22.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z23.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z24.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z21.s, #0\n" + "mov z26.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z22.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "incw %[temp], all, mul #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z23.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z24.s, #0\n" + "ptrue p7.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "whilelt p2.s, %[temp], %[width]\n" "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "mov z25.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "mov z26.s, #0\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "mov z27.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z8.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z9.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "zip2 z11.b, z11.b, z12.b\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "udot z24.s, z8.b, z2.b[0]\n" @@ -2361,24 +2238,23 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" @@ -2386,12 +2262,13 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -2491,8 +2368,11 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip1 z14.b, z14.b, z12.b\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -2571,7 +2451,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2601,7 +2481,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -2631,7 +2511,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2660,31 +2540,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2706,33 +2586,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2754,33 +2634,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2802,33 +2682,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2850,34 +2730,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -2942,7 +2824,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2972,7 +2854,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3002,7 +2884,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3031,31 +2913,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3077,33 +2959,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3125,33 +3007,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3173,33 +3055,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3221,7 +3103,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -3240,7 +3122,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -3253,175 +3135,109 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "whilelt p6.b, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "whilelt p4.b, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" + "add a_ptr3, a_ptr2, %[lda]\n" "mov z19.s, #0\n" - "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "mov z20.s, #0\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "mov z21.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "mov z22.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "whilelt p6.b, %[temp], %[leftovers]\n" "mov z23.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "whilelt p0.s, %[temp], %[width]\n" "mov z24.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "whilelt p4.b, %[temp], %[width]\n" "mov z25.s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" - "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "incw %[temp], all, mul #1\n" "mov z26.s, #0\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "mov z27.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip2 z9.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "mov z28.s, #0\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "mov z29.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" + "zip2 z11.b, z8.b, z9.b\n" + "incw %[temp], all, mul #1\n" + "zip1 z9.b, z8.b, z9.b\n" + "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "mov z30.s, #0\n" - "zip2 z15.b, z12.b, z13.b\n" - "zip1 z13.b, z12.b, z13.b\n" + "ptrue p7.b\n" "mov z31.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "zip2 z12.b, z10.b, z8.b\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "incw %[temp], all, mul #1\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "zip1 z8.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip2 z9.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z11.b, z11.b, z12.b\n" "add a_ptr3, a_ptr3, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1rqb z6.b, p7/z, [a_ptr2]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z7.b, p7/z, [a_ptr3]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p7/z, [a_ptr2]\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z8.b, z14.b, z12.b\n" - "ld1rqb z7.b, p7/z, [a_ptr3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "add a_ptr3, a_ptr3, #0x20\n" "udot z26.s, z10.b, z2.b[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -3649,29 +3465,29 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z28.s, z8.b, z3.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "udot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p7/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3797,9 +3613,13 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr3, a_ptr3, #2\n" "zip1 z14.b, z14.b, z12.b\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -3893,7 +3713,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3927,7 +3747,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3961,7 +3781,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3994,31 +3814,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4044,33 +3864,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4096,33 +3916,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4148,33 +3968,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4200,37 +4020,40 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z28.s, z8.b, z3.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "udot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p6/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" @@ -4309,7 +4132,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4343,7 +4166,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -4377,7 +4200,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4410,31 +4233,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4460,33 +4283,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4512,33 +4335,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4564,33 +4387,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4616,7 +4439,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -4641,11 +4464,12 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp deleted file mode 100644 index e2cc1d14e2..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp +++ /dev/null @@ -1,4264 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __ARM_FEATURE_SVE - -#include <algorithm> - - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -namespace arm_gemm { - -void sve_smallK_fp32_mla_1VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); - - const long loops_count = M / 4; - const long oddrow_count = M % 4; - const long ldab = lda * sizeof(float); - const long ldcb = ldc * sizeof(float); - const long odd_depth = K % 4; - const float *betaptr = β - long ldbb = ldb * sizeof(float); - - for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) { - const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1)); - long loops = loops_count; - long oddrows = oddrow_count; - long temp = 0; - const float *b_ptr0 = B + x0; - - const float *a_ptr0 = A; - - float *c_ptr0 = C + x0; - - switch(K) { - case 1: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 2: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 3: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 4: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 5: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 6: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 7: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 8: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 9: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 10: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 11: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 12: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 13: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 14: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 15: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 16: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 17: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 18: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 19: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 20: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 21: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 22: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 23: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z26.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - default: - case 24: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z26.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z27.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "fmla z28.s, z27.s, z0.s[3]\n" - "fmla z29.s, z27.s, z1.s[3]\n" - "fmla z30.s, z27.s, z2.s[3]\n" - "fmla z31.s, z27.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "fmla z28.s, z27.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - } - } -} - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp deleted file mode 100644 index 3e7e713106..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp +++ /dev/null @@ -1,4004 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __ARM_FEATURE_SVE - -#include <algorithm> - - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -namespace arm_gemm { - -void sve_smallK_hybrid_fp32_mla_1VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); - - const long loops_count = M / 4; - const long oddrow_count = M % 4; - const long ldab = lda * sizeof(float); - const long ldcb = ldc * sizeof(float); - const int K_stride = K; - const long odd_depth = K % 4; - const float *betaptr = β - - for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) { - const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1)); - long loops = loops_count; - long oddrows = oddrow_count; - long temp = 0; - const float *b_ptr0 = B + (K_stride * x0); - - const float *a_ptr0 = A; - - float *c_ptr0 = C + x0; - - switch(K) { - case 1: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 2: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 3: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 4: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 5: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 6: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 7: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 8: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 9: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 10: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 11: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 12: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 13: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 14: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 15: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 16: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 17: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 18: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 19: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 20: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 21: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 22: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 23: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - default: - case 24: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z27.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "fmla z28.s, z27.s, z0.s[3]\n" - "fmla z29.s, z27.s, z1.s[3]\n" - "fmla z30.s, z27.s, z2.s[3]\n" - "fmla z31.s, z27.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "fmla z28.s, z27.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - } - } -} - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp index ca90ee18ce..fc18cbdbbf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp @@ -31,7 +31,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_1VLx8 { @@ -39,7 +39,7 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -57,15 +57,27 @@ public: return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_smallK_hybrid_s8s32_dot_1VLx8; - smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) - { - - } + smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp index 04605b2ac1..d9813a5cdf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp @@ -25,13 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { +void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)get_vector_length<int32_t>()) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp index 453301417d..51d3e736ed 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp @@ -31,7 +31,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_1VLx8 { @@ -39,7 +39,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -57,15 +57,27 @@ public: return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_smallK_hybrid_u8u32_dot_1VLx8; - smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) - { - - } + smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp index 7c965d38a6..6eed457152 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp @@ -25,13 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { +void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)get_vector_length<uint32_t>()) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); diff --git a/src/core/NEON/kernels/arm_gemm/mergeresults.cpp b/src/core/NEON/kernels/arm_gemm/mergeresults.cpp new file mode 100644 index 0000000000..83d6bccf2b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/mergeresults.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017-2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* As some of the merges need these headers, but are all included in the + * arm_gemm namespace, put these headers here. */ +#include <algorithm> + +#include <arm_neon.h> + +#include "arm_gemm.hpp" +#include "asmlib.hpp" +#include "utils.hpp" + +namespace arm_gemm { + +template<unsigned int twidth, unsigned int height, bool sve=false, typename Tin, typename Tout> +void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append) { + // For SVE cases, multiply the width up by the vector length. + // Use the *input* type to determine this, since this will be what the kernel operated on. + const int width = twidth * (sve ? get_vector_length<Tin>() : 1); + + const int full_y_blocks = (ymax - y0) / height; + const int y_remainder = (ymax - y0) % height; + const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0); + + const int full_x_blocks = (xmax - x0) / width; + const int x_remainder = (xmax - x0) % width; + const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0); + + for (int y_block = 0; y_block < y_blocks; y_block++) { + int ybase = y0 + (y_block * height); + + int fill_rows = (y_block < full_y_blocks) ? height : y_remainder; + + for (int x_block = 0; x_block < x_blocks; x_block++) { + int xbase = x0 + (x_block * width); + + int fill_cols = (x_block < full_x_blocks) ? width : x_remainder; + + for (int row=0; row < fill_rows; row++) { + for (int col=0; col < fill_cols; col++) { + Tout &r = out[(ybase + row) * ldc + xbase + col]; + Tout v = in[row * width + col]; + + if (append) { + v += r; + } + + if (bias) { + v += bias[xbase + col]; + } + + switch(act.type) { + default: + case Activation::Type::None: + break; + + case Activation::Type::ReLU: + v = std::max(v, static_cast<Tout>(0)); + break; + + case Activation::Type::BoundedReLU: + v = std::max(std::min(v, static_cast<Tout>(act.param1)), static_cast<Tout>(0)); + break; + } + + r = v; + } + } + + in += (width * height); + } + } +} + +#include "merges/list.hpp" + +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) +template void MergeResults<12u, 8u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool); +#endif + +#if defined(__arm__) && defined(__ARM_FP16_ARGS) +template void MergeResults<8u, 6u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool); +#endif + +} // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/mergeresults.hpp b/src/core/NEON/kernels/arm_gemm/mergeresults.hpp index 04d1343b1c..1aedd23a4b 100644 --- a/src/core/NEON/kernels/arm_gemm/mergeresults.hpp +++ b/src/core/NEON/kernels/arm_gemm/mergeresults.hpp @@ -23,58 +23,9 @@ */ #pragma once -/* As some of the merges need these headers, but are all included in the - * arm_gemm namespace, put these headers here. */ -#include <arm_neon.h> - -#include "asmlib.hpp" -#include "utils.hpp" - namespace arm_gemm { template<unsigned int twidth, unsigned int height, bool sve=false, typename Tin, typename Tout> -inline void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout alpha, const Tout beta) { - // For SVE cases, multiply the width up by the vector length. - // Use the *input* type to determine this, since this will be what the kernel operated on. - const int width = twidth * (sve ? get_vector_length<Tin>() : 1); - - const int full_y_blocks = (ymax - y0) / height; - const int y_remainder = (ymax - y0) % height; - const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0); - - const int full_x_blocks = (xmax - x0) / width; - const int x_remainder = (xmax - x0) % width; - const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0); - - for (int y_block = 0; y_block < y_blocks; y_block++) { - int ybase = y0 + (y_block * height); - - int fill_rows = (y_block < full_y_blocks) ? height : y_remainder; - - for (int x_block = 0; x_block < x_blocks; x_block++) { - int xbase = x0 + (x_block * width); - - int fill_cols = (x_block < full_x_blocks) ? width : x_remainder; - - for (int row=0; row < fill_rows; row++) { - for (int col=0; col < fill_cols; col++) { - Tout &p = out[(ybase + row) * ldc + xbase + col]; - - // Special case for beta==0 - don't read the input; - // (0 * x == 0) is not always true for FP types. - if (beta == static_cast<Tout>(0)) { - p = (alpha * in[row * width + col]); - } else { - p = (p * beta) + (alpha * in[row * width + col]); - } - } - } - - in += (width * height); - } - } -} - -#include "merges/list.hpp" +void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp b/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp index e1af2d4490..9409646818 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp @@ -28,13 +28,35 @@ #include <arm_neon.h> template<> -inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) { +void MergeResults<8, 6, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) { const float *inptr = in; prefetch_6x(inptr); prefetch_6x(inptr + 96); - float32x4_t av = vdupq_n_f32(alpha); - float32x4_t bv = vdupq_n_f32(beta); + float nullbias[8]; + float minval = - std::numeric_limits<float>::infinity(); + float maxval = std::numeric_limits<float>::infinity(); + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + float32x4_t minv = vdupq_n_f32(minval); + float32x4_t maxv = vdupq_n_f32(maxval); + + if (!append && !bias) + { + memset(nullbias, 0, (8 * sizeof(float))); + } for (int y=y0; y<ymax; y+=8) { float *outptr0 = out + (y * ldout) + x0; @@ -61,16 +83,12 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo switch ((y + 5) - ymax) { case 4: outptr1 = dummyres; - // fall through case 3: outptr2 = dummyres; - // fall through case 2: outptr3 = dummyres; - // fall through case 1: outptr4 = dummyres; - // fall through case 0: outptr5 = dummyres; break; @@ -80,24 +98,24 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo } } - if (beta == 0.0f) { - /* If beta=0, don't read the original input at all. */ + if (append) { + /* Append mode: Read, activate, write. */ /* For ragged X, manually copy over the valid results. */ if ((i+7) >= xmax) { for (int xi=0; xi<8; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 8]); + *outptr1 = std::min(std::max(minval, inptr[xi + 8] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 16]); + *outptr2 = std::min(std::max(minval, inptr[xi + 16] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 24]); + *outptr3 = std::min(std::max(minval, inptr[xi + 24] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 32]); + *outptr4 = std::min(std::max(minval, inptr[xi + 32] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 40]); + *outptr5 = std::min(std::max(minval, inptr[xi + 40] + *outptr5), maxval); outptr5++; } } @@ -107,69 +125,100 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo __asm __volatile ( // Rows 0-1 "VLD1.32 {d0-d3}, [%[inptr]]!\n" + "VLD1.32 {d8-d11}, [%[outptr0]]\n" "VLD1.32 {d4-d7}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[outptr1]]\n" - "VMUL.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[inptr], #352]") - "VMUL.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q2\n" + "VADD.f32 q7, q7, q3\n" ASM_PREFETCH("[%[inptr], #416]") - "VMUL.f32 q6, q2, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[inptr], #480]") - "VMUL.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr1]]!\n" // Rows 2-3 "VLD1.32 {d0-d3}, [%[inptr]]!\n" + "VLD1.32 {d8-d11}, [%[outptr2]]\n" "VLD1.32 {d4-d7}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[outptr3]]\n" - "VMUL.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr0], #96]") - "VMUL.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q2\n" + "VADD.f32 q7, q7, q3\n" ASM_PREFETCH("[%[outptr1], #96]") - "VMUL.f32 q6, q2, %q[av]\n" - ASM_PREFETCH("[%[outptr2], #96]") - "VMUL.f32 q7, q3, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" + ASM_PREFETCH("[%[outptr2], #128]") + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr3]]!\n" // Rows 4-5 "VLD1.32 {d0-d3}, [%[inptr]]!\n" + "VLD1.32 {d8-d11}, [%[outptr4]]\n" "VLD1.32 {d4-d7}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[outptr5]]\n" - "VMUL.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr3], #96]") - "VMUL.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr4]]!\n" - ASM_PREFETCH("[%[outptr4], #96]") - "VMUL.f32 q6, q2, %q[av]\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q2\n" + "VADD.f32 q7, q7, q3\n" + ASM_PREFETCH("[%[outptr4], #128]") + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[outptr5], #128]") - "VMUL.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr4]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr5]]!\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7" + : [minv] "w" (minv), [maxv] "w" (maxv) + : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "memory" ); } } else { - /* Non-zero beta: Read output and apply beta. */ + /* Bias mode: Add bias to everything, then min/max/write as before. */ + const float *biasptr = bias ? bias + i : nullbias; /* For ragged X, manually copy over the valid results. */ if ((i+7) >= xmax) { - for (int xi=0; xi<8; xi++) { + for (int xi=0; xi<7; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 8]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 8] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 16]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 16] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 24]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 32]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 32] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 40]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 40] + biasptr[xi]), maxval); outptr5++; } } @@ -178,68 +227,75 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo /* Optimized routine to copy an entire block */ __asm __volatile ( // Rows 0-1 - "VLD1.32 {d8-d11}, [%[outptr0]]\n" - "VMUL.f32 q4, q4, %q[bv]\n" - "VLD1.32 {d12-d15}, [%[outptr1]]\n" - "VMUL.f32 q5, q5, %q[bv]\n" - "VLD1.32 {d0-d3}, [%[inptr]]!\n" - "VMUL.f32 q6, q6, %q[bv]\n" - "VLD1.32 {d4-d7}, [%[inptr]]!\n" - "VMUL.f32 q7, q7, %q[bv]\n" + "VLD1.32 {d8-d11}, [%[inptr]]!\n" + "VLD1.32 {d0-d3}, [%[biasptr]]\n" + "VLD1.32 {d12-d15}, [%[inptr]]!\n" - "VMLA.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[inptr], #352]") - "VMLA.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q0\n" + "VADD.f32 q7, q7, q1\n" ASM_PREFETCH("[%[inptr], #416]") - "VMLA.f32 q6, q2, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[inptr], #480]") - "VMLA.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr1]]!\n" // Rows 2-3 - "VLD1.32 {d8-d11}, [%[outptr2]]\n" - "VMUL.f32 q4, q4, %q[bv]\n" - "VLD1.32 {d12-d15}, [%[outptr3]]\n" - "VMUL.f32 q5, q5, %q[bv]\n" - "VLD1.32 {d0-d3}, [%[inptr]]!\n" - "VMUL.f32 q6, q6, %q[bv]\n" - "VLD1.32 {d4-d7}, [%[inptr]]!\n" - "VMUL.f32 q7, q7, %q[bv]\n" + "VLD1.32 {d8-d11}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[inptr]]!\n" - "VMLA.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr0], #96]") - "VMLA.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q0\n" + "VADD.f32 q7, q7, q1\n" ASM_PREFETCH("[%[outptr1], #96]") - "VMLA.f32 q6, q2, %q[av]\n" - ASM_PREFETCH("[%[outptr2], #96]") - "VMLA.f32 q7, q3, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" + ASM_PREFETCH("[%[outptr2], #128]") + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr3]]!\n" // Rows 4-5 - "VLD1.32 {d8-d11}, [%[outptr4]]\n" - "VMUL.f32 q4, q4, %q[bv]\n" - "VLD1.32 {d12-d15}, [%[outptr5]]\n" - "VMUL.f32 q5, q5, %q[bv]\n" - "VLD1.32 {d0-d3}, [%[inptr]]!\n" - "VMUL.f32 q6, q6, %q[bv]\n" - "VLD1.32 {d4-d7}, [%[inptr]]!\n" - "VMUL.f32 q7, q7, %q[bv]\n" + "VLD1.32 {d8-d11}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[inptr]]!\n" - "VMLA.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr3], #96]") - "VMLA.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr4]]!\n" - ASM_PREFETCH("[%[outptr4], #96]") - "VMLA.f32 q6, q2, %q[av]\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q0\n" + "VADD.f32 q7, q7, q1\n" + ASM_PREFETCH("[%[outptr4], #128]") + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[outptr5], #128]") - "VMLA.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr4]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr5]]!\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7" + : [minv] "w" (minv), [maxv] "w" (maxv), [biasptr] "r" (biasptr) + : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "memory" ); } } diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp deleted file mode 100644 index 9fca4e3a84..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -template<> -inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) { - const float *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 96); - - float32x4_t av = vdupq_n_f32(alpha); - float32x4_t bv = vdupq_n_f32(beta); - - for (int y=y0; y<ymax; y+=8) { - float *outptr0 = out + (y * ldout) + x0; - float *outptr1 = outptr0 + ldout; - float *outptr2 = outptr1 + ldout; - float *outptr3 = outptr2 + ldout; - float *outptr4 = outptr3 + ldout; - float *outptr5 = outptr4 + ldout; - float *outptr6 = outptr5 + ldout; - float *outptr7 = outptr6 + ldout; - - prefetch_2x(outptr0); - prefetch_2x(outptr1); - prefetch_2x(outptr2); - prefetch_2x(outptr3); - prefetch_2x(outptr4); - prefetch_2x(outptr5); - prefetch_2x(outptr6); - prefetch_2x(outptr7); - - for (int i=x0; i<xmax; i+=12) { - float dummyres[12]; - - /* Make sure we throw away results if Y isn't a multiple of 8. - * We do this by pointing the result pointer at a dummy buffer - * we later discard. */ - if ((y+7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (beta==0.0f) { - /* If beta==0, don't read the original input at all. */ - - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDP q0, q1, [%[inptr]]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr0]], #32\n" - ASM_PREFETCH("[%[inptr], #768]") - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr0]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr1]], #32\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr1]], #16\n" - - // Rows 2-3 - "LDP q0, q1, [%[inptr], #96]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr2]], #32\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr2]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr3]], #32\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr3]], #16\n" - - // Rows 4-5 - "LDP q0, q1, [%[inptr], #192]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr4]], #32\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr4]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr5]], #32\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - "LDP q0, q1, [%[inptr], #288]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr6]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } else { - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDP q16, q17, [%[outptr0]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr0], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr1]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr1], #32]\n" - ASM_PREFETCH("[%[inptr], #768]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr]]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr0]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr0]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr1]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr1]], #16\n" - - // Rows 2-3 - "LDP q16, q17, [%[outptr2]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr2], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr3]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr3], #32]\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #96]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr2]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr2]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr3]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr3]], #16\n" - - // Rows 4-5 - ASM_PREFETCH("[%[outptr0], #80]") - "LDP q16, q17, [%[outptr4]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr4], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr5]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr5], #32]\n" - ASM_PREFETCH("[%[outptr1], #80]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #192]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr2], #80]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr4]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr4]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr3], #80]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr5]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - ASM_PREFETCH("[%[outptr4], #80]") - "LDP q16, q17, [%[outptr6]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr6], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr7]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr7], #32]\n" - ASM_PREFETCH("[%[outptr5], #80]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #288]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr6], #128]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr6]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr7], #128]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } - } - } -} - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp deleted file mode 100644 index 0e638eef1c..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -// This should be possible on any AArch64 target, but some old compilers don't support __fp16 arguments. -#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) - -#include <arm_neon.h> - -template<> -inline void MergeResults<12,8,false>(__fp16 *out, const float *in, int ldout, int y0, int ymax, int x0, int xmax, const __fp16 alpha, const __fp16 beta) { - const float *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 24); - - float32x4_t av = vdupq_n_f32(alpha); - float32x4_t bv = vdupq_n_f32(beta); - - for (int y=y0; y<ymax; y+=8) { - __fp16 *outptr0 = out + (y * ldout) + x0; - __fp16 *outptr1 = outptr0 + ldout; - __fp16 *outptr2 = outptr1 + ldout; - __fp16 *outptr3 = outptr2 + ldout; - __fp16 *outptr4 = outptr3 + ldout; - __fp16 *outptr5 = outptr4 + ldout; - __fp16 *outptr6 = outptr5 + ldout; - __fp16 *outptr7 = outptr6 + ldout; - - prefetch_2x(outptr0); - prefetch_2x(outptr1); - prefetch_2x(outptr2); - prefetch_2x(outptr3); - prefetch_2x(outptr4); - prefetch_2x(outptr5); - prefetch_2x(outptr6); - prefetch_2x(outptr7); - - for (int i=x0; i<xmax; i+=12) { - __fp16 dummyres[12]; - - /* Make sure we throw away results if Y isn't a multiple of 8. - * We do this by pointing the result pointer at a dummy buffer - * we later discard. */ - if ((y+7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (beta == ((__fp16)0.0f)) { - /* If beta==0, don't read the output. */ - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDP q0, q1, [%[inptr]]\n" - "LDP q2, q3, [%[inptr], #32]\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #768]") - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #832]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[inptr], #896]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr0]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr0]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr1]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr1]], #8\n" - - // Rows 2-3 - "LDP q0, q1, [%[inptr], #96]\n" - "LDP q2, q3, [%[inptr], #128]\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[outptr0], #64]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[outptr1], #64]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr2]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr2]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr3]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr3]], #8\n" - - // Rows 4-5 - "LDP q0, q1, [%[inptr], #192]\n" - "LDP q2, q3, [%[inptr], #224]\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr2], #64]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[outptr3], #64]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[outptr4], #88]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr4]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr4]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr5]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr5]], #8\n" - - // Rows 6-7 - "LDP q0, q1, [%[inptr], #288]\n" - "LDP q2, q3, [%[inptr], #320]\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr5], #64]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[outptr6], #88]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[outptr7], #88]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr6]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr6]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr7]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr7]], #8\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } else { - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDR q16, [%[outptr0]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr0], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr1]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr1], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr]]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[inptr], #768]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr0]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr0]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr1]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr1]], #8\n" - - // Rows 2-3 - "LDR q16, [%[outptr2]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr2], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr3]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr3], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #96]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr0], #64]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr1], #64]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr2]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr2]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr3]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr3]], #8\n" - - // Rows 4-5 - "LDR q16, [%[outptr4]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr4], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr5]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr5], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #192]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[outptr2], #64]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[outptr3], #64]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr4], #88]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr4]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr4]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr5]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr5]], #8\n" - - // Rows 6-7 - "LDR q16, [%[outptr6]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr6], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr7]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr7], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #288]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[outptr5], #64]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[outptr6], #88]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr7], #88]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr6]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr6]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr7]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr7]], #8\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } - } - } -} - -#endif // __aarch64__ && __ARM_FP16_ARGS diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp new file mode 100644 index 0000000000..7bfab412ca --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp @@ -0,0 +1,2074 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) + +template<> +void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const __fp16 *bias, Activation act, bool append) +{ + const __fp16 *inptr = in; + __fp16 nullbias[24] = { 0 }; + __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity()); + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (24 * sizeof(__fp16))); + } + + for (int y=y0; y<ymax; y+=8) + { + __fp16 *outptr0 = out + (y * ldout) + x0; + __fp16 *outptr1 = outptr0 + ldout; + __fp16 *outptr2 = outptr1 + ldout; + __fp16 *outptr3 = outptr2 + ldout; + __fp16 *outptr4 = outptr3 + ldout; + __fp16 *outptr5 = outptr4 + ldout; + __fp16 *outptr6 = outptr5 + ldout; + __fp16 *outptr7 = outptr6 + ldout; + + const int height = ymax - y; + + for (int i=x0; i<xmax; i+=24) + { + if (append) + { + switch(height) + { + case 1: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( +#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + ".arch armv8.2-a+fp16\n" +#endif + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr0]]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q13, [%[outptr1]]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q15, [%[outptr4], #0x10]\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval); + outptr5++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "str q17, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q10, [%[outptr5], #0x10]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add %[inptr], %[inptr], #0x180\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + *outptr6)), maxval); + outptr6++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "str q17, [%[outptr5]]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr6]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q13, [%[outptr6], #0x10]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q14, [%[outptr6], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "add %[inptr], %[inptr], #0x180\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + *outptr6)), maxval); + outptr6++; + *outptr7 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 168] + *outptr7)), maxval); + outptr7++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr7]]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0x150]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr7], #0x10]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr5]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0x160]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr7], #0x20]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x170]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr6]]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr6], #0x10]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "str q14, [%[outptr6], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q15, [%[outptr7]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q16, [%[outptr7], #0x10]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q17, [%[outptr7], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add %[outptr7], %[outptr7], #0x30\n" + "add %[inptr], %[inptr], #0x180\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + else + { + const __fp16 *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q13, [%[outptr0]]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr1]]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "str q19, [%[outptr2]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr1]]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr3]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "str q19, [%[outptr2]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr3]]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr4]]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval); + outptr5++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "ldr q13, [%[inptr], #0x100]\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr3]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x110]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fadd v20.8h, v20.8h, v2.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "str q17, [%[outptr4]]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fadd v13.8h, v13.8h, v3.8h\n" + "fadd v14.8h, v14.8h, v4.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr5]]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + biasptr[xi])), maxval); + outptr6++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr3]]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.8h, v20.8h, v2.8h\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.8h, v13.8h, v3.8h\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr4]]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.8h, v14.8h, v4.8h\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "fadd v15.8h, v15.8h, v2.8h\n" + "str q19, [%[outptr4], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q20, [%[outptr5]]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fadd v16.8h, v16.8h, v3.8h\n" + "fadd v17.8h, v17.8h, v4.8h\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr6]]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + biasptr[xi])), maxval); + outptr6++; + *outptr7 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 168] + biasptr[xi])), maxval); + outptr7++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr3]]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.8h, v20.8h, v2.8h\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.8h, v13.8h, v3.8h\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr4]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.8h, v14.8h, v4.8h\n" + "ldr q18, [%[inptr], #0x150]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q19, [%[outptr4], #0x20]\n" + "fadd v15.8h, v15.8h, v2.8h\n" + "ldr q19, [%[inptr], #0x160]\n" + "fadd v16.8h, v16.8h, v3.8h\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q20, [%[outptr5]]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q20, [%[inptr], #0x170]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v17.8h, v17.8h, v4.8h\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fadd v18.8h, v18.8h, v2.8h\n" + "str q14, [%[outptr5], #0x20]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "add %[outptr5], %[outptr5], #0x30\n" + "fadd v19.8h, v19.8h, v3.8h\n" + "str q15, [%[outptr6]]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fadd v20.8h, v20.8h, v4.8h\n" + "str q16, [%[outptr6], #0x10]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "str q17, [%[outptr6], #0x20]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "add %[outptr6], %[outptr6], #0x30\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ && (FP16_KERNELS || __ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp index fcdca59bdd..0c5aa7eae8 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp @@ -26,11 +26,33 @@ #ifdef __aarch64__ template<> -inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) +void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) { const float *inptr = in; + float nullbias[12] = { 0 }; + float minval = - std::numeric_limits<float>::infinity(); + float maxval = std::numeric_limits<float>::infinity(); - for (int y=y0; y<ymax; y+=8) { + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (12 * sizeof(float))); + } + + for (int y=y0; y<ymax; y+=8) + { float *outptr0 = out + (y * ldout) + x0; float *outptr1 = outptr0 + ldout; float *outptr2 = outptr1 + ldout; @@ -42,19 +64,21 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld const int height = ymax - y; - for (int i=x0; i<xmax; i+=12) { - if (beta==0.0f) + for (int i=x0; i<xmax; i+=12) + { + if (append) { - switch(height) { + switch(height) + { case 1: { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; } } @@ -62,23 +86,34 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "add %[inptr], %[inptr], #0x180\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr0]]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" "add %[outptr0], %[outptr0], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -88,13 +123,13 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; } } @@ -102,35 +137,55 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q7, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q13, [%[outptr1]]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "str q15, [%[outptr1], #0x20]\n" "add %[outptr1], %[outptr1], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -140,15 +195,15 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; } } @@ -156,47 +211,76 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr0], #0x10]\n" - "ldr q4, [%[inptr], #0x40]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q5, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q7, [%[inptr], #0x50]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q4, [%[inptr], #0x80]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v10.4s, v10.4s, v2.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" "add %[outptr2], %[outptr2], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -206,17 +290,17 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; } } @@ -224,58 +308,96 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q5, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q6, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q6, [%[inptr], #0x80]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v13.4s, v13.4s, v5.4s\n" "str q10, [%[outptr2], #0x20]\n" - "ldr q7, [%[inptr], #0xb0]\n" + "fmax v11.4s, v11.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "add %[outptr3], %[outptr3], #0x30\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr3]]\n" "add %[inptr], %[inptr], #0x180\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -285,19 +407,19 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; } } @@ -305,70 +427,117 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q6, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr1], #0x10]\n" - "ldr q7, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr2], #0x10]\n" - "ldr q4, [%[inptr], #0xa0]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr3], #0x10]\n" - "ldr q5, [%[inptr], #0xd0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr4], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q7, [%[inptr], #0x50]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q4, [%[inptr], #0x80]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q5, [%[inptr], #0xb0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr3]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q6, [%[inptr], #0xe0]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" - "add %[outptr4], %[outptr4], #0x30\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q15, [%[outptr4], #0x10]\n" "add %[inptr], %[inptr], #0x180\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -378,21 +547,21 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval); outptr5++; } } @@ -400,82 +569,138 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q6, [%[inptr], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q7, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q4, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x10]\n" - "ldr q5, [%[inptr], #0xa0]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x10]\n" - "ldr q6, [%[inptr], #0xd0]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x10]\n" - "ldr q7, [%[inptr], #0x100]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.4s, v15.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q6, [%[inptr], #0x80]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.4s, v17.4s, v9.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q7, [%[inptr], #0xb0]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q4, [%[inptr], #0xe0]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "str q17, [%[outptr5]]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q5, [%[inptr], #0x110]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q10, [%[outptr5], #0x10]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q11, [%[outptr5], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -485,23 +710,23 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + *outptr6), maxval); outptr6++; } } @@ -509,94 +734,159 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q6, [%[inptr], #0x120]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q7, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr0], #0x10]\n" - "ldr q4, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q5, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q6, [%[inptr], #0xa0]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr3], #0x10]\n" - "ldr q7, [%[inptr], #0xd0]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr4], #0x10]\n" - "ldr q4, [%[inptr], #0x100]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr5], #0x10]\n" - "ldr q5, [%[inptr], #0x130]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr6], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q7, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr3]]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.4s, v12.4s, v4.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q4, [%[inptr], #0x80]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.4s, v10.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q5, [%[inptr], #0xb0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "str q17, [%[outptr5]]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v12.4s, v12.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q6, [%[inptr], #0xe0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" - "ldr q7, [%[inptr], #0x110]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr6]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q13, [%[outptr6], #0x10]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x20]\n" - "ldr q4, [%[inptr], #0x140]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr6], #0x20]\n" + "str q14, [%[outptr6], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "add %[outptr6], %[outptr6], #0x30\n" "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -607,25 +897,25 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + *outptr6), maxval); outptr6++; - *outptr7 = (alpha * inptr[xi + 84]); + *outptr7 = std::min(std::max(minval, inptr[xi + 84] + *outptr7), maxval); outptr7++; } } @@ -633,105 +923,179 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q6, [%[inptr], #0x120]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q7, [%[inptr], #0x150]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7]]\n" - "ldr q4, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q5, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q6, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q4, [%[inptr], #0xd0]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x10]\n" - "ldr q5, [%[inptr], #0x100]\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x10]\n" - "ldr q6, [%[inptr], #0x130]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x10]\n" - "ldr q7, [%[inptr], #0x160]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr7]]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0x150]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr7], #0x10]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr5]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0x160]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr7], #0x20]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x170]\n" + "fmin v13.4s, v13.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q6, [%[inptr], #0x80]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v15.4s, v15.4s, v7.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q7, [%[inptr], #0xb0]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr6]]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr6], #0x10]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "str q14, [%[outptr6], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q4, [%[inptr], #0xe0]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q15, [%[outptr7]]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q5, [%[inptr], #0x110]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q16, [%[outptr7], #0x10]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" - "ldr q6, [%[inptr], #0x140]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q17, [%[outptr7], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x20]\n" - "ldr q7, [%[inptr], #0x170]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "add %[outptr6], %[outptr6], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" "add %[outptr7], %[outptr7], #0x30\n" "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -742,16 +1106,23 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { - switch(height) { + const float *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; } } @@ -759,29 +1130,34 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x10]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "add %[outptr0], %[outptr0], #0x30\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" "add %[inptr], %[inptr], #0x180\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -791,13 +1167,13 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; } } @@ -805,47 +1181,52 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x10]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q11, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x40]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q13, [%[outptr0]]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr1]]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" "add %[outptr1], %[outptr1], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -855,15 +1236,15 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; } } @@ -871,65 +1252,70 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x10]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr0], #0x10]\n" - "ldr q8, [%[outptr1], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x40]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q9, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x70]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q11, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x50]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q8, [%[outptr2], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr1], #0x20]\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x80]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "str q19, [%[outptr2]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" "add %[outptr2], %[outptr2], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -939,17 +1325,17 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; } } @@ -957,82 +1343,87 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr0], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x10]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q9, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x40]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q10, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x70]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q11, [%[outptr3], #0x10]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q10, [%[outptr2], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr1]]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x80]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q11, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr2], #0x20]\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xb0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr3]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" "add %[outptr3], %[outptr3], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1042,19 +1433,19 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; } } @@ -1062,100 +1453,105 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr0], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x10]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q10, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x40]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr1], #0x10]\n" - "ldr q11, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x70]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr2], #0x10]\n" - "ldr q8, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xa0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr3], #0x10]\n" - "ldr q9, [%[outptr4], #0x10]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xd0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr4], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q11, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x50]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q8, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x80]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q9, [%[outptr3], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "str q19, [%[outptr2]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xb0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q10, [%[outptr4], #0x20]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr3]]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fadd v18.4s, v18.4s, v3.4s\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "str q16, [%[outptr3], #0x20]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xe0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr4]]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" "add %[outptr4], %[outptr4], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1165,21 +1561,21 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval); outptr5++; } } @@ -1187,118 +1583,123 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr5]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q10, [%[outptr0], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x10]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q11, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x40]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q8, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x70]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x10]\n" - "ldr q9, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xa0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x10]\n" - "ldr q10, [%[outptr4], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xd0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x10]\n" - "ldr q11, [%[outptr5], #0x10]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x100]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q10, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x80]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q11, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "fadd v18.4s, v18.4s, v3.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xb0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q8, [%[outptr4], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr3]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x110]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fadd v20.4s, v20.4s, v2.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xe0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q9, [%[outptr5], #0x20]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "str q17, [%[outptr4]]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fadd v13.4s, v13.4s, v3.4s\n" + "fadd v14.4s, v14.4s, v4.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q19, [%[outptr4], #0x20]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x110]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr5]]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1308,23 +1709,23 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + biasptr[xi]), maxval); outptr6++; } } @@ -1332,136 +1733,141 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr5]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q10, [%[outptr6]]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x120]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q11, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x10]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr0], #0x10]\n" - "ldr q8, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x40]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q9, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x70]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q10, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xa0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr3], #0x10]\n" - "ldr q11, [%[outptr4], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xd0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr4], #0x10]\n" - "ldr q8, [%[outptr5], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x100]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr5], #0x10]\n" - "ldr q9, [%[outptr6], #0x10]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x130]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr6], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q11, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x50]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q8, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x80]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q9, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xb0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q10, [%[outptr4], #0x20]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr3]]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.4s, v18.4s, v3.4s\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.4s, v20.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.4s, v13.4s, v3.4s\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xe0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" - "ldr q11, [%[outptr5], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr4]]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.4s, v14.4s, v4.4s\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "fadd v15.4s, v15.4s, v2.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x110]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x20]\n" - "ldr q8, [%[outptr6], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q20, [%[outptr5]]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fadd v16.4s, v16.4s, v3.4s\n" + "fadd v17.4s, v17.4s, v4.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q14, [%[outptr5], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x140]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr6], #0x20]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr6]]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" "add %[outptr6], %[outptr6], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1472,25 +1878,25 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + biasptr[xi]), maxval); outptr6++; - *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta); + *outptr7 = std::min(std::max(minval, inptr[xi + 84] + biasptr[xi]), maxval); outptr7++; } } @@ -1498,153 +1904,158 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr5]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q10, [%[outptr6]]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x120]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q11, [%[outptr7]]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x150]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7]]\n" - "ldr q8, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x10]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q9, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x40]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q10, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x70]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q11, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q8, [%[outptr4], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xd0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x10]\n" - "ldr q9, [%[outptr5], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x100]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x10]\n" - "ldr q10, [%[outptr6], #0x10]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x130]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x10]\n" - "ldr q11, [%[outptr7], #0x10]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x160]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q10, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x80]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q11, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xb0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q8, [%[outptr4], #0x20]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr3]]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.4s, v18.4s, v3.4s\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.4s, v20.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.4s, v13.4s, v3.4s\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xe0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q9, [%[outptr5], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr4]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.4s, v14.4s, v4.4s\n" + "ldr q18, [%[inptr], #0x150]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "fadd v15.4s, v15.4s, v2.4s\n" + "ldr q19, [%[inptr], #0x160]\n" + "fadd v16.4s, v16.4s, v3.4s\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x110]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" - "ldr q10, [%[outptr6], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q20, [%[outptr5]]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q20, [%[inptr], #0x170]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v17.4s, v17.4s, v4.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fadd v18.4s, v18.4s, v2.4s\n" + "str q14, [%[outptr5], #0x20]\n" + "fmin v17.4s, v17.4s, v0.4s\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x140]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x20]\n" - "ldr q11, [%[outptr7], #0x20]\n" + "fadd v19.4s, v19.4s, v3.4s\n" + "str q15, [%[outptr6]]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fadd v20.4s, v20.4s, v4.4s\n" + "str q16, [%[outptr6], #0x10]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "str q17, [%[outptr6], #0x20]\n" + "fmax v19.4s, v19.4s, v1.4s\n" "add %[outptr6], %[outptr6], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x170]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" "add %[outptr7], %[outptr7], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp deleted file mode 100644 index 60cc2f32da..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -// AArch64 only, and either the FP16_KERNELS option set or the target explicitly supports FP16 vectors. -#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) - -template<> -inline void MergeResults<24, 8>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, - const int x0, const int xmax, const __fp16 alpha, const __fp16 beta) { - const __fp16 *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 48); - - float16x8_t va = vdupq_n_f16(alpha); - float16x8_t vb = vdupq_n_f16(beta); - - for (int y=y0; y<ymax; y+=8) { - __fp16 *outptr0 = out + (y * ldout) + x0; - __fp16 *outptr1 = outptr0 + ldout; - __fp16 *outptr2 = outptr1 + ldout; - __fp16 *outptr3 = outptr2 + ldout; - __fp16 *outptr4 = outptr3 + ldout; - __fp16 *outptr5 = outptr4 + ldout; - __fp16 *outptr6 = outptr5 + ldout; - __fp16 *outptr7 = outptr6 + ldout; - - prefetch_2x(outptr0); - prefetch_2x(outptr1); - prefetch_2x(outptr2); - prefetch_2x(outptr3); - prefetch_2x(outptr4); - prefetch_2x(outptr5); - prefetch_2x(outptr6); - prefetch_2x(outptr7); - - for (int i=x0; i<xmax; i+=24) { - __fp16 dummyres[24]; - - /* Make sure we throw away results if Y isn't a multiple of 8. - * We do this by pointing the result pointer at a dummy buffer - * we later discard. */ - if ((y+7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - - } - } - - if (beta == (__fp16)0.0f) { - /* If beta===0, don't read the output. */ - - /* For ragged X, manually copy over the valid results. */ - if ((i+23) >= xmax) { - for (int xi=0; xi<24; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); - outptr0++; - *outptr1 = (alpha * inptr[xi + 24]); - outptr1++; - *outptr2 = (alpha * inptr[xi + 48]); - outptr2++; - *outptr3 = (alpha * inptr[xi + 72]); - outptr3++; - *outptr4 = (alpha * inptr[xi + 96]); - outptr4++; - *outptr5 = (alpha * inptr[xi + 120]); - outptr5++; - *outptr6 = (alpha * inptr[xi + 144]); - outptr6++; - *outptr7 = (alpha * inptr[xi + 168]); - outptr7++; - } - } - inptr += 192; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( -#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - ".arch armv8.2-a+fp16\n" -#endif - // Rows 0-1 - ASM_PREFETCH("[%[inptr], #768]") - "LDP q0, q1, [%[inptr]]\n" - "LDP q2, q3, [%[inptr], #32]\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr0]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr0]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr1]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr1]], #16\n" - ASM_PREFETCH("[%[inptr], #960]") - - // Rows 2-3 - ASM_PREFETCH("[%[inptr], #1024]") - "LDP q0, q1, [%[inptr], #96]\n" - "LDP q2, q3, [%[inptr], #128]\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr2]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr2]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr0], #80]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr3]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr3]], #16\n" - ASM_PREFETCH("[%[outptr1], #80]") - - // Rows 4-5 - ASM_PREFETCH("[%[outptr2], #80]") - "LDP q0, q1, [%[inptr], #192]\n" - "LDP q2, q3, [%[inptr], #224]\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr3], #80]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr4]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr4]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr4], #80]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr5]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - ASM_PREFETCH("[%[outptr5], #80]") - "LDP q0, q1, [%[inptr], #288]\n" - "LDP q2, q3, [%[inptr], #320]\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr6], #128]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr6]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr7], #128]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [va] "w" (va), [vb] "w" (vb) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } else { - /* For ragged X, manually copy over the valid results. */ - if ((i+23) >= xmax) { - for (int xi=0; xi<24; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (alpha * inptr[xi + 24]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (alpha * inptr[xi + 48]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (alpha * inptr[xi + 72]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (alpha * inptr[xi + 96]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (alpha * inptr[xi + 120]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (alpha * inptr[xi + 144]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (alpha * inptr[xi + 168]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 192; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( -#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - ".arch armv8.2-a+fp16\n" -#endif - // Rows 0-1 - "LDP q16, q17, [%[outptr0]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr0], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr1]]\n" - "FMUL v18.8h, v18.8h, %[vb].8h\n" - ASM_PREFETCH("[%[inptr], #768]") - "LDR q21, [%[outptr1], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr]]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr0]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr0]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr1]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr1]], #16\n" - ASM_PREFETCH("[%[inptr], #960]") - - // Rows 2-3 - "LDP q16, q17, [%[outptr2]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr2], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr3]]\n" - "FMUL v18.8h, v18.8h, %[vb].8h\n" - ASM_PREFETCH("[%[inptr], #1024]") - "LDR q21, [%[outptr3], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr], #96]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr2]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr2]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr0], #80]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr3]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr3]], #16\n" - ASM_PREFETCH("[%[outptr1], #80]") - - // Rows 4-5 - "LDP q16, q17, [%[outptr4]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr4], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr5]]\n" - "FMUL v18.8h, v18.8h, %[vb].8h\n" - ASM_PREFETCH("[%[outptr2], #80]") - "LDR q21, [%[outptr5], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr], #192]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr3], #80]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr4]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr4]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr4], #80]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr5]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - "LDP q16, q17, [%[outptr6]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr6], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr7]]\n" - ASM_PREFETCH("[%[outptr5], #80]") - "FMUL v18.8h, v18.8h, %[vb].8h\n" - "LDR q21, [%[outptr7], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr], #288]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr6], #128]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr6]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr7], #128]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [va] "w" (va), [vb] "w" (vb) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } - } - } -} - -#endif // __aarch64__ && (FP16_KERNELS || __ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp deleted file mode 100644 index 410a0a1dc9..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright (c) 2017-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -template<> -inline void MergeResults<12, 8, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t alpha, const int32_t beta) { - UNUSED(alpha); - const int32_t *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 96); - - for (int y=y0; y<ymax; y+=8) { - int32_t *outptr0 = out + (y * ldout) + x0; - int32_t *outptr1 = outptr0 + ldout; - int32_t *outptr2 = outptr1 + ldout; - int32_t *outptr3 = outptr2 + ldout; - int32_t *outptr4 = outptr3 + ldout; - int32_t *outptr5 = outptr4 + ldout; - int32_t *outptr6 = outptr5 + ldout; - int32_t *outptr7 = outptr6 + ldout; - - prefetch_2x(outptr0); - prefetch_2x(outptr1); - prefetch_2x(outptr2); - prefetch_2x(outptr3); - prefetch_2x(outptr4); - prefetch_2x(outptr5); - prefetch_2x(outptr6); - prefetch_2x(outptr7); - - for (int i=x0; i<xmax; i+=12) { - int32_t dummyres[12]; - - /* Make sure we throw away results if Y isn't a multiple of 8. - * We do this by pointing the result pointer at a dummy buffer - * we later discard. */ - if ((y+7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (inptr[xi + 12]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (inptr[xi + 24]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (inptr[xi + 36]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (inptr[xi + 48]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (inptr[xi + 60]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (inptr[xi + 72]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (inptr[xi + 84]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 96; - } else { - if (beta == 0u) { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Row 0 - ASM_PREFETCH("[%x[outptr1], #192]") - "ldr q0, [%x[inptr]]\n" - "ldr q1, [%x[inptr], #0x10]\n" - "ldr q2, [%x[inptr], #0x20]\n" - - // Row 1 - ASM_PREFETCH("[%x[outptr2], #192]") - "ldr q3, [%x[inptr], #0x30]\n" - "str q0, [%x[outptr0]], #0x10\n" - "ldr q4, [%x[inptr], #0x40]\n" - "str q1, [%x[outptr0]], #0x10\n" - "ldr q5, [%x[inptr], #0x50]\n" - "str q2, [%x[outptr0]], #0x10\n" - - // Row 2 - ASM_PREFETCH("[%x[outptr3], #192]") - "ldr q0, [%x[inptr], #0x60]\n" - "str q3, [%x[outptr1]], #0x10\n" - "ldr q1, [%x[inptr], #0x70]\n" - "str q4, [%x[outptr1]], #0x10\n" - "ldr q2, [%x[inptr], #0x80]\n" - "str q5, [%x[outptr1]], #0x10\n" - - // Row 3 - ASM_PREFETCH("[%x[outptr4], #192]") - "ldr q3, [%x[inptr], #0x90]\n" - "str q0, [%x[outptr2]], #0x10\n" - "ldr q4, [%x[inptr], #0xa0]\n" - "str q1, [%x[outptr2]], #0x10\n" - "ldr q5, [%x[inptr], #0xb0]\n" - "str q2, [%x[outptr2]], #0x10\n" - - // Row 4 - ASM_PREFETCH("[%x[outptr5], #192]") - "ldr q0, [%x[inptr], #0xc0]\n" - "str q3, [%x[outptr3]], #0x10\n" - "ldr q1, [%x[inptr], #0xd0]\n" - "str q4, [%x[outptr3]], #0x10\n" - "ldr q2, [%x[inptr], #0xe0]\n" - "str q5, [%x[outptr3]], #0x10\n" - - // Row 5 - ASM_PREFETCH("[%x[outptr6], #192]") - "ldr q3, [%x[inptr], #0xf0]\n" - "str q0, [%x[outptr4]], #0x10\n" - "ldr q4, [%x[inptr], #0x100]\n" - "str q1, [%x[outptr4]], #0x10\n" - "ldr q5, [%x[inptr], #0x110]\n" - "str q2, [%x[outptr4]], #0x10\n" - - // Row 6 - ASM_PREFETCH("[%x[outptr7], #192]") - "ldr q0, [%x[inptr], #0x120]\n" - "str q3, [%x[outptr5]], #0x10\n" - "ldr q1, [%x[inptr], #0x130]\n" - "str q4, [%x[outptr5]], #0x10\n" - "ldr q2, [%x[inptr], #0x140]\n" - "str q5, [%x[outptr5]], #0x10\n" - - // Row 7 - "ldr q3, [%x[inptr], #0x150]\n" - "str q0, [%x[outptr6]], #0x10\n" - "ldr q4, [%x[inptr], #0x160]\n" - "str q1, [%x[outptr6]], #0x10\n" - "ldr q5, [%x[inptr], #0x170]\n" - "str q2, [%x[outptr6]], #0x10\n" - "str q3, [%x[outptr7]], #0x10\n" - "str q4, [%x[outptr7]], #0x10\n" - "str q5, [%x[outptr7]], #0x10\n" - - "add %x[inptr], %x[inptr], #0x180\n" - : [outptr0] "+r" (outptr0), - [outptr1] "+r" (outptr1), - [outptr2] "+r" (outptr2), - [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), - [outptr5] "+r" (outptr5), - [outptr6] "+r" (outptr6), - [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6" - ); - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Row 0 - ASM_PREFETCH("[%x[outptr1], #192]") - "ldr q3, [%x[outptr0]]\n" - "ldr q4, [%x[outptr0], #0x10]\n" - "ldr q5, [%x[outptr0], #0x20]\n" - "ldr q6, [%x[inptr]]\n" - "ldr q7, [%x[inptr], #0x10]\n" - "ldr q8, [%x[inptr], #0x20]\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr1]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr1], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr1], #0x20]\n" - - // Row 1 - ASM_PREFETCH("[%x[outptr2], #192]") - "ldr q6, [%x[inptr], #0x30]\n" - "str q3, [%x[outptr0]], #0x10\n" - "ldr q7, [%x[inptr], #0x40]\n" - "str q4, [%x[outptr0]], #0x10\n" - "ldr q8, [%x[inptr], #0x50]\n" - "str q5, [%x[outptr0]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "ldr q3, [%x[outptr2]]\n" - "add v1.4s, v1.4s, v7.4s\n" - "ldr q4, [%x[outptr2], #0x10]\n" - "add v2.4s, v2.4s, v8.4s\n" - "ldr q5, [%x[outptr2], #0x20]\n" - - // Row 2 - ASM_PREFETCH("[%x[outptr3], #192]") - "ldr q6, [%x[inptr], #0x60]\n" - "str q0, [%x[outptr1]], #0x10\n" - "ldr q7, [%x[inptr], #0x70]\n" - "str q1, [%x[outptr1]], #0x10\n" - "ldr q8, [%x[inptr], #0x80]\n" - "str q2, [%x[outptr1]], #0x10\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr3]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr3], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr3], #0x20]\n" - - // Row 3 - ASM_PREFETCH("[%x[outptr4], #192]") - "ldr q6, [%x[inptr], #0x90]\n" - "str q3, [%x[outptr2]], #0x10\n" - "ldr q7, [%x[inptr], #0xa0]\n" - "str q4, [%x[outptr2]], #0x10\n" - "ldr q8, [%x[inptr], #0xb0]\n" - "str q5, [%x[outptr2]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "ldr q3, [%x[outptr4]]\n" - "add v1.4s, v1.4s, v7.4s\n" - "ldr q4, [%x[outptr4], #0x10]\n" - "add v2.4s, v2.4s, v8.4s\n" - "ldr q5, [%x[outptr4], #0x20]\n" - - // Row 4 - ASM_PREFETCH("[%x[outptr5], #192]") - "ldr q6, [%x[inptr], #0xc0]\n" - "str q0, [%x[outptr3]], #0x10\n" - "ldr q7, [%x[inptr], #0xd0]\n" - "str q1, [%x[outptr3]], #0x10\n" - "ldr q8, [%x[inptr], #0xe0]\n" - "str q2, [%x[outptr3]], #0x10\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr5]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr5], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr5], #0x20]\n" - - // Row 5 - ASM_PREFETCH("[%x[outptr6], #192]") - "ldr q6, [%x[inptr], #0xf0]\n" - "str q3, [%x[outptr4]], #0x10\n" - "ldr q7, [%x[inptr], #0x100]\n" - "str q4, [%x[outptr4]], #0x10\n" - "ldr q8, [%x[inptr], #0x110]\n" - "str q5, [%x[outptr4]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "ldr q3, [%x[outptr6]]\n" - "add v1.4s, v1.4s, v7.4s\n" - "ldr q4, [%x[outptr6], #0x10]\n" - "add v2.4s, v2.4s, v8.4s\n" - "ldr q5, [%x[outptr6], #0x20]\n" - - // Row 6 - ASM_PREFETCH("[%x[outptr7], #192]") - "ldr q6, [%x[inptr], #0x120]\n" - "str q0, [%x[outptr5]], #0x10\n" - "ldr q7, [%x[inptr], #0x130]\n" - "str q1, [%x[outptr5]], #0x10\n" - "ldr q8, [%x[inptr], #0x140]\n" - "str q2, [%x[outptr5]], #0x10\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr7]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr7], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr7], #0x20]\n" - - // Row 7 - "ldr q6, [%x[inptr], #0x150]\n" - "str q3, [%x[outptr6]], #0x10\n" - "ldr q7, [%x[inptr], #0x160]\n" - "str q4, [%x[outptr6]], #0x10\n" - "ldr q8, [%x[inptr], #0x170]\n" - "str q5, [%x[outptr6]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "add v1.4s, v1.4s, v7.4s\n" - "add v2.4s, v2.4s, v8.4s\n" - "str q0, [%x[outptr7]], #0x10\n" - "str q1, [%x[outptr7]], #0x10\n" - "str q2, [%x[outptr7]], #0x10\n" - - "add %x[inptr], %x[inptr], #0x180\n" - : [outptr0] "+r" (outptr0), - [outptr1] "+r" (outptr1), - [outptr2] "+r" (outptr2), - [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), - [outptr5] "+r" (outptr5), - [outptr6] "+r" (outptr6), - [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8" - ); - - } - } - } - } -} - -template<> -inline void MergeResults<12, 8>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t alpha, const uint32_t beta) { - // Since the above code uses only MUL and MLA instructions discard the "unsignedness" and proceed safely. - MergeResults<12, 8>(reinterpret_cast<int32_t*>(out), reinterpret_cast<const int32_t*>(in), ldout, y0, ymax, x0, xmax, static_cast<const int32_t>(alpha), static_cast<const int32_t>(beta)); -} - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp new file mode 100644 index 0000000000..313f8295ff --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp @@ -0,0 +1,1595 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<12, 8, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const int32_t *inptr = in; + int32_t nullbias[12] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (12 * sizeof(int32_t))); + } + + for (int y=y0; y<ymax; y+=8) + { + int32_t *outptr0 = out + (y * ldout) + x0; + int32_t *outptr1 = outptr0 + ldout; + int32_t *outptr2 = outptr1 + ldout; + int32_t *outptr3 = outptr2 + ldout; + int32_t *outptr4 = outptr3 + ldout; + int32_t *outptr5 = outptr4 + ldout; + int32_t *outptr6 = outptr5 + ldout; + int32_t *outptr7 = outptr6 + ldout; + + const int height = ymax - y; + + for (int i=x0; i<xmax; i+=12) + { + if (append) + { + switch(height) + { + case 1: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "ldr q11, [%[inptr], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "add %[inptr], %[inptr], #0x180\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q13, [%[outptr1]]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "str q17, [%[outptr2], #0x10]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q14, [%[inptr], #0x40]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q3, [%[outptr3]]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr3]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "str q15, [%[outptr4], #0x10]\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q16, [%[outptr2]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "str q10, [%[outptr5], #0x10]\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "str q16, [%[outptr2]]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "str q13, [%[outptr6], #0x10]\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + *outptr7 += inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "ldr q7, [%[outptr7]]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "ldr q15, [%[inptr], #0x150]\n" + "ldr q8, [%[outptr7], #0x10]\n" + "ldr q16, [%[inptr], #0x160]\n" + "str q13, [%[outptr6], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr7], #0x20]\n" + "ldr q17, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr7]]\n" + "str q16, [%[outptr7], #0x10]\n" + "str q17, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + else + { + const int32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "ldr q13, [%[inptr]]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v2.4s\n" + "add v14.4s, v14.4s, v3.4s\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x30]\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v14.4s, v14.4s, v3.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "add v16.4s, v16.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "str q14, [%[outptr0], #0x10]\n" + "add v17.4s, v17.4s, v3.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "add v19.4s, v19.4s, v2.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "add v14.4s, v14.4s, v2.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v3.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "str q17, [%[outptr1], #0x10]\n" + "ldr q17, [%[inptr], #0xc0]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + *outptr7 = biasptr[xi] + inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "add v18.4s, v18.4s, v3.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "str q20, [%[outptr2], #0x10]\n" + "ldr q20, [%[inptr], #0xf0]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "ldr q18, [%[inptr], #0x150]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v18.4s, v18.4s, v2.4s\n" + "ldr q19, [%[inptr], #0x160]\n" + "str q20, [%[outptr5]]\n" + "ldr q20, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v19.4s, v19.4s, v3.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "add v20.4s, v20.4s, v4.4s\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp new file mode 100644 index 0000000000..a93060fa5a --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<4, 4, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const int32_t *inptr = in; + int32_t nullbias[4] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (4 * sizeof(int32_t))); + } + + for (int y=y0; y<ymax; y+=4) + { + int32_t *outptr0 = out + (y * ldout) + x0; + int32_t *outptr1 = outptr0 + ldout; + int32_t *outptr2 = outptr1 + ldout; + int32_t *outptr3 = outptr2 + ldout; + + const int height = ymax - y; + + for (int i=x0; i<xmax; i+=4) + { + if (append) + { + switch(height) + { + case 1: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + *outptr3 += inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr3]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "add %[inptr], %[inptr], #0x40\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q13, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + else + { + const int32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v13.4s, v13.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "ldr q14, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add v13.4s, v13.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v14.4s, v14.4s, v2.4s\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "prfm PSTL1KEEP, [%[outptr3], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "str q14, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp new file mode 100644 index 0000000000..5569f5157b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp @@ -0,0 +1,1595 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<12, 8, false>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const uint32_t *inptr = in; + uint32_t nullbias[12] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (12 * sizeof(uint32_t))); + } + + for (int y=y0; y<ymax; y+=8) + { + uint32_t *outptr0 = out + (y * ldout) + x0; + uint32_t *outptr1 = outptr0 + ldout; + uint32_t *outptr2 = outptr1 + ldout; + uint32_t *outptr3 = outptr2 + ldout; + uint32_t *outptr4 = outptr3 + ldout; + uint32_t *outptr5 = outptr4 + ldout; + uint32_t *outptr6 = outptr5 + ldout; + uint32_t *outptr7 = outptr6 + ldout; + + const int height = ymax - y; + + for (int i=x0; i<xmax; i+=12) + { + if (append) + { + switch(height) + { + case 1: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "ldr q11, [%[inptr], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "add %[inptr], %[inptr], #0x180\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q13, [%[outptr1]]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "str q17, [%[outptr2], #0x10]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q14, [%[inptr], #0x40]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q3, [%[outptr3]]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr3]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "str q15, [%[outptr4], #0x10]\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q16, [%[outptr2]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "str q10, [%[outptr5], #0x10]\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "str q16, [%[outptr2]]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "str q13, [%[outptr6], #0x10]\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + *outptr7 += inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "ldr q7, [%[outptr7]]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "ldr q15, [%[inptr], #0x150]\n" + "ldr q8, [%[outptr7], #0x10]\n" + "ldr q16, [%[inptr], #0x160]\n" + "str q13, [%[outptr6], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr7], #0x20]\n" + "ldr q17, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr7]]\n" + "str q16, [%[outptr7], #0x10]\n" + "str q17, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + else + { + const uint32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "ldr q13, [%[inptr]]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v2.4s\n" + "add v14.4s, v14.4s, v3.4s\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x30]\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v14.4s, v14.4s, v3.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "add v16.4s, v16.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "str q14, [%[outptr0], #0x10]\n" + "add v17.4s, v17.4s, v3.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "add v19.4s, v19.4s, v2.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "add v14.4s, v14.4s, v2.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v3.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "str q17, [%[outptr1], #0x10]\n" + "ldr q17, [%[inptr], #0xc0]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + *outptr7 = biasptr[xi] + inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "add v18.4s, v18.4s, v3.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "str q20, [%[outptr2], #0x10]\n" + "ldr q20, [%[inptr], #0xf0]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "ldr q18, [%[inptr], #0x150]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v18.4s, v18.4s, v2.4s\n" + "ldr q19, [%[inptr], #0x160]\n" + "str q20, [%[outptr5]]\n" + "ldr q20, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v19.4s, v19.4s, v3.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "add v20.4s, v20.4s, v4.4s\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp new file mode 100644 index 0000000000..fd01bb2392 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<4, 4, false>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const uint32_t *inptr = in; + uint32_t nullbias[4] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (4 * sizeof(uint32_t))); + } + + for (int y=y0; y<ymax; y+=4) + { + uint32_t *outptr0 = out + (y * ldout) + x0; + uint32_t *outptr1 = outptr0 + ldout; + uint32_t *outptr2 = outptr1 + ldout; + uint32_t *outptr3 = outptr2 + ldout; + + const int height = ymax - y; + + for (int i=x0; i<xmax; i+=4) + { + if (append) + { + switch(height) + { + case 1: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + *outptr3 += inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr3]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "add %[inptr], %[inptr], #0x40\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q13, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + else + { + const uint32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v13.4s, v13.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "ldr q14, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add v13.4s, v13.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v14.4s, v14.4s, v2.4s\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "prfm PSTL1KEEP, [%[outptr3], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "str q14, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/list.hpp b/src/core/NEON/kernels/arm_gemm/merges/list.hpp index 788a957d5e..4edb497967 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/list.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/list.hpp @@ -22,10 +22,13 @@ * SOFTWARE. */ #include "a32_merge_float_8x6.hpp" -#include "a64_merge_float_12x8.hpp" -#include "a64_merge_float_to_half_12x8.hpp" -#include "a64_merge_half_24x8.hpp" -#include "a64_merge_int32_12x8.hpp" +#include "a64_merge_fp16_24x8.hpp" +#include "a64_merge_fp32_12x8.hpp" +#include "a64_merge_s32_12x8.hpp" +#include "a64_merge_s32_4x4.hpp" +#include "a64_merge_u32_12x8.hpp" +#include "a64_merge_u32_4x4.hpp" +#include "sve_merge_fp16_3VLx8.hpp" #include "sve_merge_fp32_3VLx8.hpp" #include "sve_merge_s32_3VLx8.hpp" #include "sve_merge_u32_3VLx8.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp new file mode 100644 index 0000000000..e5efc09b52 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp @@ -0,0 +1,1879 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __ARM_FEATURE_SVE + +template<> +void MergeResults<3, 8, true>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const __fp16 *bias, Activation act, bool append) +{ + const __fp16 *inptr = in; + __fp16 nullbias[384] = { 0 }; + __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity()); + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + + for (int y=y0; y<ymax; y+=8) + { + __fp16 *outptr0 = out + (y * ldout) + x0; + __fp16 *outptr1 = outptr0 + ldout; + __fp16 *outptr2 = outptr1 + ldout; + __fp16 *outptr3 = outptr2 + ldout; + __fp16 *outptr4 = outptr3 + ldout; + __fp16 *outptr5 = outptr4 + ldout; + __fp16 *outptr6 = outptr5 + ldout; + __fp16 *outptr7 = outptr6 + ldout; + + const int height = ymax - y; + + for (int i=x0; i<xmax; i+=(3 * get_vector_length<__fp16>())) + { + if (append) + { + switch(height) + { + case 1: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "fadd z12.h, z12.h, z4.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 2: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "fadd z14.h, z14.h, z6.h\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "addvl %[outptr1], %[outptr1], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 3: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z16.h, z16.h, z8.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fadd z17.h, z17.h, z9.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "addvl %[outptr2], %[outptr2], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 4: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z12.h, z12.h, z4.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "addvl %[outptr3], %[outptr3], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 5: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z14.h, z14.h, z6.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z15.h, z15.h, z7.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z16.h, z16.h, z8.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 6: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "ld1h z9.h, p0/z, [%[outptr5]]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z10.h, p1/z, [x8]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fadd z17.h, z17.h, z9.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "fmin z10.h, p1/m, z10.h, z0.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z11.h, p2/m, z11.h, z0.h\n" + "st1h z17.h, p0, [%[outptr5]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z10.h, p1/m, z10.h, z1.h\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z11.h, p2/m, z11.h, z1.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 7: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "ld1h z9.h, p0/z, [%[outptr5]]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z10.h, p1/z, [x8]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z4.h, p0/z, [%[outptr6]]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "ld1h z5.h, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "ld1h z13.h, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z6.h, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "fmin z10.h, p1/m, z10.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z10.h, p1/m, z10.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z11.h, p2/m, z11.h, z0.h\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd z13.h, z13.h, z5.h\n" + "st1h z17.h, p0, [%[outptr5]]\n" + "fmax z11.h, p2/m, z11.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z12.h, p0/m, z12.h, z0.h\n" + "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n" + "fmax z12.h, p0/m, z12.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z12.h, p0, [%[outptr6]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "addvl %[outptr4], %[outptr4], #3\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z13.h, p1, [%[outptr6], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "addvl %[outptr5], %[outptr5], #3\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "st1h z14.h, p2, [%[outptr6], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "addvl %[outptr6], %[outptr6], #3\n" + "addvl %[inptr], %[inptr], #24\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + default: + case 8: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "ld1h z9.h, p0/z, [%[outptr5]]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z10.h, p1/z, [x8]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z4.h, p0/z, [%[outptr6]]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "ld1h z5.h, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "ld1h z13.h, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z6.h, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "fmin z10.h, p1/m, z10.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p0/z, [%[outptr7]]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p0/z, [x8, #5, MUL VL]\n" + "fmax z10.h, p1/m, z10.h, z1.h\n" + "ld1h z8.h, p1/z, [%[outptr7], #1, MUL VL]\n" + "fmin z11.h, p2/m, z11.h, z0.h\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p1/z, [x8, #6, MUL VL]\n" + "fmin z12.h, p0/m, z12.h, z0.h\n" + "ld1h z9.h, p2/z, [%[outptr7], #2, MUL VL]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p0, [%[outptr5]]\n" + "fmax z11.h, p2/m, z11.h, z1.h\n" + "ld1h z17.h, p2/z, [x8, #7, MUL VL]\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z12.h, p0/m, z12.h, z1.h\n" + "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z12.h, p0, [%[outptr6]]\n" + "fadd z16.h, z16.h, z8.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z15.h, p0/m, z15.h, z0.h\n" + "st1h z13.h, p1, [%[outptr6], #1, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.h, p1/m, z16.h, z0.h\n" + "st1h z14.h, p2, [%[outptr6], #2, MUL VL]\n" + "fmax z15.h, p0/m, z15.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z17.h, p2/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z16.h, p1/m, z16.h, z1.h\n" + "st1h z15.h, p0, [%[outptr7]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z17.h, p2/m, z17.h, z1.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "st1h z16.h, p1, [%[outptr7], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "addvl %[outptr5], %[outptr5], #3\n" + "st1h z17.h, p2, [%[outptr7], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "addvl %[outptr6], %[outptr6], #3\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "addvl %[outptr7], %[outptr7], #3\n" + "addvl %[inptr], %[inptr], #24\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + + } + } + else + { + const __fp16 *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "fadd z13.h, z13.h, z2.h\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "fadd z15.h, z15.h, z4.h\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 2: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "fadd z15.h, z15.h, z4.h\n" + "fadd z16.h, z16.h, z2.h\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fadd z17.h, z17.h, z3.h\n" + "fadd z18.h, z18.h, z4.h\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "addvl %[outptr1], %[outptr1], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 3: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "fadd z20.h, z20.h, z3.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.h, z13.h, z4.h\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "addvl %[outptr2], %[outptr2], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 4: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.h, z13.h, z4.h\n" + "fadd z14.h, z14.h, z2.h\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "fadd z15.h, z15.h, z3.h\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "addvl %[outptr3], %[outptr3], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 5: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z16.h, z16.h, z4.h\n" + "fadd z17.h, z17.h, z2.h\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "fadd z18.h, z18.h, z3.h\n" + "fadd z19.h, z19.h, z4.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 6: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.h, z17.h, z2.h\n" + "ld1h z13.h, p1/z, [x8]\n" + "fadd z18.h, z18.h, z3.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z19.h, z19.h, z4.h\n" + "fadd z20.h, z20.h, z2.h\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmin z20.h, p0/m, z20.h, z0.h\n" + "fadd z13.h, z13.h, z3.h\n" + "fadd z14.h, z14.h, z4.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "fmax z20.h, p0/m, z20.h, z1.h\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "st1h z20.h, p0, [%[outptr5]]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n" + "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 7: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.h, z17.h, z2.h\n" + "ld1h z13.h, p1/z, [x8]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.h, z18.h, z3.h\n" + "fadd z19.h, z19.h, z4.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "ld1h z15.h, p0/z, [x8, #2, MUL VL]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.h, z20.h, z2.h\n" + "ld1h z16.h, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.h, z13.h, z3.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "ld1h z17.h, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.h, p0/m, z20.h, z0.h\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.h, z14.h, z4.h\n" + "fadd z15.h, z15.h, z2.h\n" + "fmax z20.h, p0/m, z20.h, z1.h\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z20.h, p0, [%[outptr5]]\n" + "fmin z15.h, p0/m, z15.h, z0.h\n" + "fadd z16.h, z16.h, z3.h\n" + "fadd z17.h, z17.h, z4.h\n" + "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "fmax z15.h, p0/m, z15.h, z1.h\n" + "fmin z16.h, p1/m, z16.h, z0.h\n" + "fmin z17.h, p2/m, z17.h, z0.h\n" + "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" + "fmax z16.h, p1/m, z16.h, z1.h\n" + "st1h z15.h, p0, [%[outptr6]]\n" + "fmax z17.h, p2/m, z17.h, z1.h\n" + "st1h z16.h, p1, [%[outptr6], #1, MUL VL]\n" + "st1h z17.h, p2, [%[outptr6], #2, MUL VL]\n" + "addvl %[outptr6], %[outptr6], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + default: + case 8: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.h, z17.h, z2.h\n" + "ld1h z13.h, p1/z, [x8]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.h, z18.h, z3.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "ld1h z15.h, p0/z, [x8, #2, MUL VL]\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "fadd z19.h, z19.h, z4.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.h, z20.h, z2.h\n" + "ld1h z16.h, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.h, z13.h, z3.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "ld1h z17.h, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.h, p0/m, z20.h, z0.h\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.h, z14.h, z4.h\n" + "ld1h z18.h, p0/z, [x8, #5, MUL VL]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "fmax z20.h, p0/m, z20.h, z1.h\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z15.h, z15.h, z2.h\n" + "ld1h z19.h, p1/z, [x8, #6, MUL VL]\n" + "fadd z16.h, z16.h, z3.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z20.h, p0, [%[outptr5]]\n" + "fmin z15.h, p0/m, z15.h, z0.h\n" + "ld1h z20.h, p2/z, [x8, #7, MUL VL]\n" + "fmin z16.h, p1/m, z16.h, z0.h\n" + "fadd z17.h, z17.h, z4.h\n" + "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n" + "fadd z18.h, z18.h, z2.h\n" + "fmax z15.h, p0/m, z15.h, z1.h\n" + "fmax z16.h, p1/m, z16.h, z1.h\n" + "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n" + "fmin z17.h, p2/m, z17.h, z0.h\n" + "addvl %[outptr5], %[outptr5], #3\n" + "fmin z18.h, p0/m, z18.h, z0.h\n" + "st1h z15.h, p0, [%[outptr6]]\n" + "fadd z19.h, z19.h, z3.h\n" + "fmax z17.h, p2/m, z17.h, z1.h\n" + "fadd z20.h, z20.h, z4.h\n" + "st1h z16.h, p1, [%[outptr6], #1, MUL VL]\n" + "fmax z18.h, p0/m, z18.h, z1.h\n" + "fmin z19.h, p1/m, z19.h, z0.h\n" + "fmin z20.h, p2/m, z20.h, z0.h\n" + "st1h z17.h, p2, [%[outptr6], #2, MUL VL]\n" + "addvl %[outptr6], %[outptr6], #3\n" + "fmax z19.h, p1/m, z19.h, z1.h\n" + "fmax z20.h, p2/m, z20.h, z1.h\n" + "st1h z18.h, p0, [%[outptr7]]\n" + "st1h z19.h, p1, [%[outptr7], #1, MUL VL]\n" + "st1h z20.h, p2, [%[outptr7], #2, MUL VL]\n" + "addvl %[outptr7], %[outptr7], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + + } + } + } + } +} + +#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp deleted file mode 100644 index 7479c8d77c..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp +++ /dev/null @@ -1,1208 +0,0 @@ -/* - * Copyright (c) 2018 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -inline void MergeResults<2, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) -{ - const float *inptr = in; - - for (int y=y0; y<ymax; y+=8) { - float *outptr0 = out + (y * ldout) + x0; - float *outptr1 = outptr0 + ldout; - float *outptr2 = outptr1 + ldout; - float *outptr3 = outptr2 + ldout; - float *outptr4 = outptr3 + ldout; - float *outptr5 = outptr4 + ldout; - float *outptr6 = outptr5 + ldout; - float *outptr7 = outptr6 + ldout; - - const int height = ymax - y; - - for (int i=x0; i<xmax; i+=(2 * get_vector_length<float>())) { - if (beta==0.0f) - { - switch(height) { - case 1: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 2: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 3: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 4: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 5: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 6: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PSTL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 7: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-5, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PSTL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PSTL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - default: - case 8: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z7.s, p0/z, [x8, #-2, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PSTL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PSTL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "prfm PSTL1KEEP, [%[outptr7], #0x40]\n" - "addvl %[outptr7], %[outptr7], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - - } - } - else - { - switch(height) { - case 1: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 2: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 3: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 4: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 5: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 6: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PLDL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 7: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PLDL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PLDL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - default: - case 8: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z11.s, p0/z, [%[outptr7]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PLDL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PLDL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "prfm PLDL1KEEP, [%[outptr7], #0x40]\n" - "addvl %[outptr7], %[outptr7], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - - } - } - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp index 4b51066ebe..f7feec4d83 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp @@ -26,11 +26,33 @@ #ifdef __ARM_FEATURE_SVE template<> -inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) +void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) { const float *inptr = in; + float nullbias[192] = { 0 }; + float minval = - std::numeric_limits<float>::infinity(); + float maxval = std::numeric_limits<float>::infinity(); - for (int y=y0; y<ymax; y+=8) { + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length<float>() * sizeof(float))); + } + + for (int y=y0; y<ymax; y+=8) + { float *outptr0 = out + (y * ldout) + x0; float *outptr1 = outptr0 + ldout; float *outptr2 = outptr1 + ldout; @@ -42,45 +64,52 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou const int height = ymax - y; - for (int i=x0; i<xmax; i+=(3 * get_vector_length<float>())) { - if (beta==0.0f) + for (int i=x0; i<xmax; i+=(3 * get_vector_length<float>())) + { + if (append) { - switch(height) { + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "fadd z12.s, z12.s, z4.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -91,47 +120,61 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "fadd z14.s, z14.s, z6.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -142,59 +185,82 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z16.s, z16.s, z8.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -205,70 +271,102 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z12.s, z12.s, z4.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -279,82 +377,123 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z14.s, z14.s, z6.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z15.s, z15.s, z7.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z16.s, z16.s, z8.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -365,94 +504,144 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z10.s, p1/z, [x8]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "fmin z10.s, p1/m, z10.s, z0.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmin z11.s, p2/m, z11.s, z0.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z10.s, p1/m, z10.s, z1.s\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z11.s, p2/m, z11.s, z1.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -463,106 +652,165 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #3, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z10.s, p1/z, [x8]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "fmin z10.s, p1/m, z10.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z10.s, p1/m, z10.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #4, MUL VL]\n" + "fmin z11.s, p2/m, z11.s, z0.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd z13.s, z13.s, z5.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "fmax z11.s, p2/m, z11.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin z12.s, p0/m, z12.s, z0.s\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "fmax z12.s, p0/m, z12.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "addvl %[outptr5], %[outptr5], #3\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -574,117 +822,185 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z7.s, p0/z, [x8, #5, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #3, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z10.s, p1/z, [x8]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "fmin z10.s, p1/m, z10.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p0/z, [%[outptr7]]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n" + "fmax z10.s, p1/m, z10.s, z1.s\n" + "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n" + "fmin z11.s, p2/m, z11.s, z0.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n" + "fmin z12.s, p0/m, z12.s, z0.s\n" + "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "fmax z11.s, p2/m, z11.s, z1.s\n" + "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z12.s, p0/m, z12.s, z1.s\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #4, MUL VL]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "fadd z16.s, z16.s, z8.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.s, p1/m, z16.s, z0.s\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" + "fmax z15.s, p0/m, z15.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z16.s, p1/m, z16.s, z1.s\n" + "st1w z15.s, p0, [%[outptr7]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "addvl %[outptr5], %[outptr5], #3\n" + "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "addvl %[outptr6], %[outptr6], #3\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -694,48 +1010,54 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou } else { - switch(height) { + const float *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z13.s, z13.s, z2.s\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "fadd z15.s, z15.s, z4.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -746,59 +1068,58 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "fadd z15.s, z15.s, z4.s\n" + "fadd z16.s, z16.s, z2.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fadd z17.s, z17.s, z3.s\n" + "fadd z18.s, z18.s, z4.s\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -809,77 +1130,76 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "fadd z20.s, z20.s, z3.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.s, z13.s, z4.s\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -890,94 +1210,93 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.s, z13.s, z4.s\n" + "fadd z14.s, z14.s, z2.s\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "fadd z15.s, z15.s, z3.s\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -988,112 +1307,111 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z16.s, z16.s, z4.s\n" + "fadd z17.s, z17.s, z2.s\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "fadd z18.s, z18.s, z3.s\n" + "fadd z19.s, z19.s, z4.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1104,130 +1422,129 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "fadd z18.s, z18.s, z3.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z19.s, z19.s, z4.s\n" + "fadd z20.s, z20.s, z2.s\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmin z20.s, p0/m, z20.s, z0.s\n" + "fadd z13.s, z13.s, z3.s\n" + "fadd z14.s, z14.s, z4.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "fmax z20.s, p0/m, z20.s, z1.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1238,148 +1555,147 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr6], #2, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.s, z18.s, z3.s\n" + "fadd z19.s, z19.s, z4.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.s, z20.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.s, z13.s, z3.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.s, p0/m, z20.s, z0.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.s, z14.s, z4.s\n" + "fadd z15.s, z15.s, z2.s\n" + "fmax z20.s, p0/m, z20.s, z1.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" + "fadd z16.s, z16.s, z3.s\n" + "fadd z17.s, z17.s, z4.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "fmax z15.s, p0/m, z15.s, z1.s\n" + "fmin z16.s, p1/m, z16.s, z0.s\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "fmax z16.s, p1/m, z16.s, z1.s\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1391,165 +1707,164 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z11.s, p0/z, [%[outptr7]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr6], #2, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr7], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.s, z18.s, z3.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "fadd z19.s, z19.s, z4.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.s, z20.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.s, z13.s, z3.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.s, p0/m, z20.s, z0.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.s, z14.s, z4.s\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "fmax z20.s, p0/m, z20.s, z1.s\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z15.s, z15.s, z2.s\n" + "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" + "fadd z16.s, z16.s, z3.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" + "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "fmin z16.s, p1/m, z16.s, z0.s\n" + "fadd z17.s, z17.s, z4.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "fadd z18.s, z18.s, z2.s\n" + "fmax z15.s, p0/m, z15.s, z1.s\n" + "fmax z16.s, p1/m, z16.s, z1.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" "addvl %[outptr5], %[outptr5], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "fmin z18.s, p0/m, z18.s, z0.s\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "fadd z19.s, z19.s, z3.s\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" + "fadd z20.s, z20.s, z4.s\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "fmax z18.s, p0/m, z18.s, z1.s\n" + "fmin z19.s, p1/m, z19.s, z0.s\n" + "fmin z20.s, p2/m, z20.s, z0.s\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "fmax z19.s, p1/m, z19.s, z1.s\n" + "fmax z20.s, p2/m, z20.s, z1.s\n" + "st1w z18.s, p0, [%[outptr7]]\n" + "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp index 2ea38a78df..03a28241fd 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp @@ -26,11 +26,20 @@ #ifdef __ARM_FEATURE_SVE template<> -inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t alpha, const int32_t beta) +void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append) { + UNUSED(act); + const int32_t *inptr = in; + int32_t nullbias[192] = { 0 }; + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length<int32_t>() * sizeof(int32_t))); + } - for (int y=y0; y<ymax; y+=8) { + for (int y=y0; y<ymax; y+=8) + { int32_t *outptr0 = out + (y * ldout) + x0; int32_t *outptr1 = outptr0 + ldout; int32_t *outptr2 = outptr1 + ldout; @@ -42,42 +51,44 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int const int height = ymax - y; - for (int i=x0; i<xmax; i+=(3 * get_vector_length<int32_t>())) { - if (beta==0) + for (int i=x0; i<xmax; i+=(3 * get_vector_length<int32_t>())) + { + if (append) { - switch(height) { + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -88,41 +99,47 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -133,50 +150,62 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -187,58 +216,76 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -249,67 +296,91 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "add z16.s, z16.s, z8.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -320,76 +391,106 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -400,85 +501,121 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -490,93 +627,135 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "ld1w z7.s, p0/z, [%[outptr7]]\n" "addvl %[outptr5], %[outptr5], #3\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n" + "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n" + "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n" + "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p0, [%[outptr7]]\n" + "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -586,45 +765,46 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int } else { - switch(height) { + const int32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z13.s, z13.s, z2.s\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -635,53 +815,44 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z4.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z16.s, z16.s, z2.s\n" + "add z17.s, z17.s, z3.s\n" + "add z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -692,68 +863,56 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z20.s, z20.s, z3.s\n" + "add z13.s, z13.s, z4.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -764,82 +923,67 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z16.s, z16.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -850,97 +994,79 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -951,112 +1077,91 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z7.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1067,127 +1172,103 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z7.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z4.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1199,141 +1280,114 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr7]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8]\n" - "ld1w z7.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z6.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z7.s, p0/z, [%[outptr7], #2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z18.s, z18.s, z2.s\n" + "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "add z19.s, z19.s, z3.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z20.s, z20.s, z4.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "st1w z18.s, p0, [%[outptr7]]\n" + "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp index eb684e2118..67a6eb32bb 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp @@ -26,11 +26,20 @@ #ifdef __ARM_FEATURE_SVE template<> -inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t alpha, const uint32_t beta) +void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append) { + UNUSED(act); + const uint32_t *inptr = in; + uint32_t nullbias[192] = { 0 }; + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length<uint32_t>() * sizeof(uint32_t))); + } - for (int y=y0; y<ymax; y+=8) { + for (int y=y0; y<ymax; y+=8) + { uint32_t *outptr0 = out + (y * ldout) + x0; uint32_t *outptr1 = outptr0 + ldout; uint32_t *outptr2 = outptr1 + ldout; @@ -42,42 +51,44 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in const int height = ymax - y; - for (int i=x0; i<xmax; i+=(3 * get_vector_length<uint32_t>())) { - if (beta==0u) + for (int i=x0; i<xmax; i+=(3 * get_vector_length<uint32_t>())) + { + if (append) { - switch(height) { + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -88,41 +99,47 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -133,50 +150,62 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -187,58 +216,76 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -249,67 +296,91 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "add z16.s, z16.s, z8.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -320,76 +391,106 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -400,85 +501,121 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -490,93 +627,135 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "ld1w z7.s, p0/z, [%[outptr7]]\n" "addvl %[outptr5], %[outptr5], #3\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n" + "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n" + "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n" + "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p0, [%[outptr7]]\n" + "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -586,45 +765,46 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in } else { - switch(height) { + const uint32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z13.s, z13.s, z2.s\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -635,53 +815,44 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z4.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z16.s, z16.s, z2.s\n" + "add z17.s, z17.s, z3.s\n" + "add z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -692,68 +863,56 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z20.s, z20.s, z3.s\n" + "add z13.s, z13.s, z4.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -764,82 +923,67 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z16.s, z16.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -850,97 +994,79 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -951,112 +1077,91 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z7.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1067,127 +1172,103 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z7.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z4.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1199,141 +1280,114 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr7]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8]\n" - "ld1w z7.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z6.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z7.s, p0/z, [%[outptr7], #2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z18.s, z18.s, z2.s\n" + "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "add z19.s, z19.s, z3.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z20.s, z20.s, z4.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "st1w z18.s, p0, [%[outptr7]]\n" + "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; diff --git a/src/core/NEON/kernels/arm_gemm/misc.cpp b/src/core/NEON/kernels/arm_gemm/misc.cpp index 7b345e2e98..6758a88c65 100644 --- a/src/core/NEON/kernels/arm_gemm/misc.cpp +++ b/src/core/NEON/kernels/arm_gemm/misc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2017-2019 ARM Limited. * * SPDX-License-Identifier: MIT * diff --git a/src/core/NEON/kernels/arm_gemm/profiler.hpp b/src/core/NEON/kernels/arm_gemm/profiler.hpp deleted file mode 100644 index 1b944c4ccd..0000000000 --- a/src/core/NEON/kernels/arm_gemm/profiler.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef CYCLE_PROFILING - -#include "../perf.h" - -#ifndef NO_MULTI_THREADING -#include <mutex> -#endif - -namespace arm_gemm { - -#ifndef NO_MULTI_THREADING -extern std::mutex report_mutex; -#endif - -class profiler { -private: - static const int maxevents = 100000; - unsigned long times[maxevents] = { }; - unsigned long units[maxevents] = { }; - int events[maxevents] = { }; - int currentevent=0; - int countfd=0; - - class ScopedProfilerClass { - private: - profiler &_parent; - bool legal=false; - - public: - ScopedProfilerClass(profiler &prof, int i, unsigned long u) : _parent(prof) { - if (prof.currentevent==maxevents) - return; - - prof.events[prof.currentevent]=i; - prof.units[prof.currentevent]=u; - legal=true; - start_counter(prof.countfd); - } - - ~ScopedProfilerClass() { - if (!legal) return; - - long long cycs = stop_counter(_parent.countfd); - _parent.times[_parent.currentevent++] = cycs; - } - }; - -public: - profiler() { - countfd=open_cycle_counter(); - } - - ~profiler() { - close(countfd); - int tots[5]; - unsigned long counts[5]; - unsigned long tunits[5]; - const char * descs[] = { "Prepare A", "Prepare B", "Kernel", "Merge" }; - - for (int i=1; i<5; i++) { - tots[i] = 0; - counts[i] = 0; - tunits[i] = 0; - } - - for (int i=0; i<currentevent; i++) { -// printf("%10s: %ld\n", descs[events[i]-1], times[i]); - tots[events[i]]++; - counts[events[i]] += times[i]; - tunits[events[i]] += units[i]; - } - -#ifdef NO_MULTI_THREADING - printf("Profiled events:\n"); -#else - std::lock_guard<std::mutex> lock(report_mutex); - printf("Profiled events (cpu %d):\n", sched_getcpu()); -#endif - - printf("%20s %9s %9s %9s %12s %9s\n", "", "Events", "Total", "Average", "Bytes/MACs", "Per cycle"); - for (int i=1; i<5; i++) { - printf("%20s: %9d %9ld %9ld %12lu %9.2f\n",descs[i-1],tots[i],counts[i],counts[i]/tots[i],tunits[i],(float)tunits[i]/counts[i]); - } - } - - template <typename T> - void operator() (int i, unsigned long u, T func) { - if (currentevent==maxevents) { - func(); - } else { - events[currentevent] = i; - units[currentevent] = u; - start_counter(countfd); - func(); - long long cycs = stop_counter(countfd); - times[currentevent++] = cycs; - } - } - - ScopedProfilerClass ScopedProfiler(int i, unsigned long u) { - return ScopedProfilerClass(*this, i, u); - } -}; - -#endif // CYCLE_PROFILING - -} // namespace arm_gemm - -#define PROFILE_PREPA 1 -#define PROFILE_PREPB 2 -#define PROFILE_KERNEL 3 -#define PROFILE_MERGE 4 diff --git a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp index 5e67bc9c0e..188dd0b06d 100644 --- a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp +++ b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp @@ -41,7 +41,7 @@ private: int32_t *_row_sums = nullptr; int32_t *_col_sums = nullptr; ARequantizeLayer32 _params; - GemmArgs<Tr> _args; + GemmArgs _args; barrier _barrier; void *working_space = nullptr; @@ -80,12 +80,13 @@ private: /* Use the first part of our working space for the subgemm result, pass the operand details straight through. */ _subgemm->set_arrays(this->_Aptr, this->_lda, this->_A_batch_stride, this->_A_multi_stride, this->_Bptr, this->_ldb, this->_B_multi_stride, - reinterpret_cast<Tgemm *>(working_space), _args._Nsize, (_args._Nsize * _args._Msize), (_args._Nsize * _args._Msize * _args._nbatches)); + reinterpret_cast<Tgemm *>(working_space), _args._Nsize, (_args._Nsize * _args._Msize), (_args._Nsize * _args._Msize * _args._nbatches), + nullptr, 0); } void col_sums_pretransposed(const To *B, const int ldb, const int B_multi_stride) { for (unsigned int multi=0; multi<_args._nmulti; multi++) { - compute_col_sums(_params, _args._Nsize, _args._Ksize, B + (multi * B_multi_stride), ldb, _col_sums + (multi * _args._Nsize), _args._Ksize, 0); + compute_col_sums(_params, _args._Nsize, _args._Ksize, B + (multi * B_multi_stride), ldb, _col_sums + (multi * _args._Nsize), _args._Ksize, multi, 0); } } @@ -94,7 +95,7 @@ private: unsigned int last_col = ((threadid + 1) * _args._Nsize) / _args._maxthreads; for (unsigned int multi=0; multi<_args._nmulti; multi++) { - compute_col_sums(_params, (last_col - first_col), _args._Ksize, this->_Bptr + (multi * this->_B_multi_stride) + first_col, this->_ldb, _col_sums + (multi * _args._Nsize) + first_col, _args._Ksize, first_col); + compute_col_sums(_params, (last_col - first_col), _args._Ksize, this->_Bptr + (multi * this->_B_multi_stride) + first_col, this->_ldb, _col_sums + (multi * _args._Nsize) + first_col, _args._Ksize, multi, first_col); } } @@ -121,8 +122,11 @@ private: public: - QuantizeWrapper(const GemmArgs<Tr> &args, const ARequantizeLayer32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) { - GemmArgs<Tgemm> newargs = GemmArgs<Tgemm>(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._trA, args._trB, 1, 0, args._maxthreads, args._pretransposed_hint, nullptr); + QuantizeWrapper(const QuantizeWrapper &) = delete; + QuantizeWrapper operator=(const QuantizeWrapper &) = delete; + + QuantizeWrapper(const GemmArgs &args, const ARequantizeLayer32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) { + GemmArgs newargs = GemmArgs(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._trA, args._trB, Activation(), args._maxthreads, args._pretransposed_hint, nullptr); _subgemm = gemm<To, Tgemm>(newargs); if (_subgemm == nullptr) { @@ -134,15 +138,11 @@ public: } } - QuantizeWrapper(const QuantizeWrapper &) = delete; - QuantizeWrapper &operator=(const QuantizeWrapper &) = delete; - QuantizeWrapper(QuantizeWrapper &&) = default; - QuantizeWrapper &operator=(QuantizeWrapper &&) = default; - void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, const int B_multi_stride, - Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override { - GemmCommon<To, Tr>::set_arrays(A, lda, A_batch_stride, A_multi_stride, B, ldb, B_multi_stride, C, ldc, C_batch_stride, C_multi_stride); + Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const Tr *bias, const int bias_multi_stride) override { + GemmCommon<To, Tr>::set_arrays(A, lda, A_batch_stride, A_multi_stride, B, ldb, B_multi_stride, C, ldc, C_batch_stride, C_multi_stride, bias, bias_multi_stride); arrays_set = true; set_child_arrays(); @@ -232,8 +232,9 @@ public: _col_sums = reinterpret_cast<int32_t *>(buffer); } - void set_quantized_bias(const int32_t *bias) override { + void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override { _params.bias = bias; + _params.bias_multi_stride = bias_multi_stride; } }; diff --git a/src/core/NEON/kernels/arm_gemm/quantized.cpp b/src/core/NEON/kernels/arm_gemm/quantized.cpp index 28f01bd252..bffb7ddcb3 100644 --- a/src/core/NEON/kernels/arm_gemm/quantized.cpp +++ b/src/core/NEON/kernels/arm_gemm/quantized.cpp @@ -495,10 +495,10 @@ namespace { * We could do 64 adds in the signed case, but that * optimization is not worth the complexity. */ - if (i > 0 && ((i & 31) == 0)) { - finalsums[r] = vpadalq_s16(finalsums[r], sums[r]); - sums[r] = vdupq_n_s16(0); - } + if (i > 0 && ((i & 31) == 0)) { + finalsums[r] = vpadalq_s16(finalsums[r], sums[r]); + sums[r] = vdupq_n_s16(0); + } sums[r] = accumulate_16(input + (r * in_stride) + (i * 16), sums[r]); } } @@ -526,6 +526,7 @@ namespace { * that the terms can simply be added in the requantize code. * */ switch (rows) { + default: case 1: /* If we only have one output, just use ADDV. Multiply * the offset into all four components separately so it @@ -567,8 +568,6 @@ namespace { vst1q_s32(row_bias, t0); break; - default: - break; } } @@ -736,12 +735,11 @@ inline void add_block(const int8_t *input, unsigned int in_stride, int32_t *outp } } - /* "first_col" parameter is used to offset the read into the qp.bias array, * in cases where we are not computing the first columns of the output (i.e. * in multithreaded cases where we divide columns across threads) */ template<typename T> -void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col) { +void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col) { memset(reinterpret_cast<void *>(col_bias), 0, width * sizeof(int32_t)); for (unsigned int row=0; row<height; row+=4) { @@ -752,6 +750,7 @@ void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned if (numcols==16) { switch(numrows) { + default: case 1: add_block<1>(input + row * in_stride + col, in_stride, col_bias + col); break; @@ -767,8 +766,6 @@ void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned case 4: add_block<4>(input + row * in_stride + col, in_stride, col_bias + col); break; - default: - break; } } else { for (; col<width; col++) { @@ -788,15 +785,15 @@ void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned result = (qp.a_offset * qp.b_offset * depth) - (result * qp.a_offset); if (qp.bias != nullptr) { - result += qp.bias[col + first_col]; + result += qp.bias[multi * qp.bias_multi_stride + col + first_col]; } col_bias[col] = result; } } -template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const int8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col); -template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const uint8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col); +template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const int8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col); +template void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const uint8_t *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/quantized.hpp b/src/core/NEON/kernels/arm_gemm/quantized.hpp index eaa303b237..a22750796c 100644 --- a/src/core/NEON/kernels/arm_gemm/quantized.hpp +++ b/src/core/NEON/kernels/arm_gemm/quantized.hpp @@ -37,6 +37,6 @@ void compute_row_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned template<typename T> void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, - unsigned int first_col); + unsigned int multi, unsigned int first_col); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp index 2c661e7fa9..c3c1e8d930 100644 --- a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp +++ b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp @@ -64,8 +64,8 @@ public: } template<typename TOut> - void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut alpha, const TOut beta) { - MergeResults<width, height>(out, in, stride, y0, ymax, x0, xmax, alpha, beta); + void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool append) { + MergeResults<width, height>(out, in, stride, y0, ymax, x0, xmax, bias, act, append); } }; diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp index b7323ebaea..6b64e5e36c 100644 --- a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp +++ b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp @@ -63,8 +63,8 @@ public: } template<typename TOut> - void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut alpha, const TOut beta) { - MergeResults<width_vectors / mmla, height, true>(out, in, stride, y0, ymax, x0, xmax, alpha, beta); + void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool append) { + MergeResults<width_vectors / mmla, height, true>(out, in, stride, y0, ymax, x0, xmax, bias, act, append); } }; diff --git a/src/core/NEON/kernels/arm_gemm/transform.hpp b/src/core/NEON/kernels/arm_gemm/transform.hpp index d790d566b6..bdae90300b 100644 --- a/src/core/NEON/kernels/arm_gemm/transform.hpp +++ b/src/core/NEON/kernels/arm_gemm/transform.hpp @@ -84,7 +84,6 @@ struct TransformImpl { *out++ = static_cast<TOut>(0); } } - // "row" tail - row is out of range so fill with zeros always. TOut zeroval = static_cast<TOut>(0); int pads = blank_rows * (fill_cols + blank_cols); diff --git a/src/core/NEON/kernels/assembly/Helpers.cpp b/src/core/NEON/kernels/assembly/Helpers.cpp index 3d8d66d7fc..93ea6c8d5e 100644 --- a/src/core/NEON/kernels/assembly/Helpers.cpp +++ b/src/core/NEON/kernels/assembly/Helpers.cpp @@ -24,16 +24,13 @@ #include "arm_compute/core/NEON/kernels/assembly/Helpers.h" -#include "arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp" - namespace arm_compute { arm_gemm::KernelDescription get_gemm_info(DataType input_type, const CPUInfo &ci, const unsigned int num_threads, const INEGEMMWrapperKernel::Params &p, - float alpha, - float beta, + arm_gemm::Activation activation, bool pretranspose_hint) { switch(input_type) @@ -42,25 +39,25 @@ arm_gemm::KernelDescription get_gemm_info(DataType in case DataType::QASYMM8: case DataType::U8: { - arm_gemm::GemmArgs<uint32_t> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method<uint8_t, uint32_t>(args); } case DataType::S8: { - arm_gemm::GemmArgs<int32_t> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method<int8_t, int32_t>(args); } #endif // __aarch64__ #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC case DataType::F16: { - arm_gemm::GemmArgs<__fp16> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method<__fp16, __fp16>(args); } #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ case DataType::F32: { - arm_gemm::GemmArgs<float> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method<float, float>(args); } default: diff --git a/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h b/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h deleted file mode 100644 index 6e30148b5d..0000000000 --- a/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ -#define __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ - -#include "../arm_gemm/utils.hpp" -#include "arm_gemm.hpp" - -#include "../arm_gemm/mergeresults.hpp" -#include "../arm_gemm/transform.hpp" - -#include "../arm_gemm/kernels/a32_sgemm_8x6.hpp" -#include "../arm_gemm/kernels/a64_gemm_s8_12x8.hpp" -#include "../arm_gemm/kernels/a64_gemm_s8_4x4.hpp" -#include "../arm_gemm/kernels/a64_gemm_u8_12x8.hpp" -#include "../arm_gemm/kernels/a64_gemm_u8_4x4.hpp" -#include "../arm_gemm/kernels/a64_hgemm_24x8.hpp" -#include "../arm_gemm/kernels/a64_sgemm_12x8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp" - -namespace arm_compute -{ -namespace detail -{ -/** GEMM Interleaved Strategy interface */ -class IInterleavedStrategy -{ -public: - /** Virtual Destructor */ - virtual ~IInterleavedStrategy() = default; - /** Return output height of the interleaved strategy - * - * @return Output height of strategy - */ - virtual unsigned int out_height() const = 0; - /** Instantiate and configure a prepareB Kernel - * - * @param[in] b Input tensor B. - * @param[in] transformed_b Reshaped tensor B. - * @param[in] params GM, N, K sizes. - * @param[in] ci CPUInfo to be used for kernel configuration. - * - * @return A wrapped specialized prepareB kernel - */ - virtual std::unique_ptr<NEGEMMInterleavedPrepareBWrapperKernel> instantiate_prepareB(const ITensor *b, - ITensor *transformed_b, - const INEGEMMWrapperKernel::Params ¶ms, - const CPUInfo &ci) = 0; - /** Instantiate and configure a transformA Kernel - * - * @param[in] a Input tensor A. - * @param[in] transformed_a Reshaped tensor A. - * @param[in] block_walker Window representing the layout of the matrix's blocks. - * @param[in] params M, N, K sizes. - * @param[in] gemm_info GEMM meta-data - * - * @return A wrapped specialized transformA kernel - */ - virtual std::unique_ptr<NEGEMMInterleavedTransformAWrapper> instantiate_transformA(const ITensor *a, - ITensor *transformed_a, - const Window &block_walker, - const INEGEMMWrapperKernel::Params ¶ms, - const GEMMInfo &gemm_info) = 0; - /** Instantiate and configure a prepareB Kernel - * - * @param[in] transformed_a Already reshaped tensor A. - * @param[in] transformed_b Already reshaped tensor B. - * @param[in] tmp_c Temporary buffer to be used to store intermediate results. - * @param[in] c Result tensor C. - * @param[in] block_walker Window containing iteration information for the M and batch dimensions. - * @param[in] block_sizes Block sizes to use for the matrix multiplication (A & B must have been reshaped using these same block sizes). - * @param[in] params M, N, K sizes. - * @param[in] alpha Alpha value - * @param[in] beta Beta value - * @param[in] gemm_info GEMM meta-data - * @param[in] num_threads Maximum number of threads that might be used for the calculations. - * - * @return A wrapped specialized MatrixMultiply kernel - */ - virtual std::unique_ptr<NEGEMMInterleavedMatrixMultiplyWrapper> instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, - const Window &block_walker, const BlockSizes &block_sizes, - const INEGEMMWrapperKernel::Params ¶ms, float alpha, float beta, const GEMMInfo &gemm_info, - unsigned int num_threads) = 0; - /** Calculates the block sizes of a given strategy - * - * @param[in] ci CPUInfo to be used for kernel configuration. - * @param[in] params M, N, K sizes. - * - * @return BlockSizes for a given strategy - */ - virtual BlockSizes calculate_block_sizes_for_strategy(const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) = 0; -}; - -/** Interleaved Strategy class */ -template <typename StrategyType> -class InterleavedStrategy : public IInterleavedStrategy -{ -public: - using strategy = StrategyType; - -public: - // Inherited methods overridden - unsigned int out_height() const override - { - return strategy::out_height(); - } - std::unique_ptr<NEGEMMInterleavedPrepareBWrapperKernel> instantiate_prepareB(const ITensor *b, - ITensor *transformed_b, - const INEGEMMWrapperKernel::Params ¶ms, - const CPUInfo &ci) override - { - auto prepare_b = support::cpp14::make_unique<NEGEMMInterleavedPrepareBWrapperKernelTemplate<strategy>>(); - prepare_b->configure(b, transformed_b, false, ci, params); - return std::move(prepare_b); - } - std::unique_ptr<NEGEMMInterleavedTransformAWrapper> instantiate_transformA(const ITensor *a, - ITensor *transformed_a, - const Window &block_walker, - const INEGEMMWrapperKernel::Params ¶ms, - const GEMMInfo &gemm_info) override - { - auto transform_a = support::cpp14::make_unique<NEGEMMInterleavedTransformAWrapperTemplate<strategy>>(); - transform_a->configure(a, transformed_a, false, gemm_info.reinterpret_input_as_3d(), block_walker, params); - return std::move(transform_a); - } - std::unique_ptr<NEGEMMInterleavedMatrixMultiplyWrapper> instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, - const Window &block_walker, const BlockSizes &block_sizes, - const INEGEMMWrapperKernel::Params ¶ms, float alpha, float beta, const GEMMInfo &gemm_info, - unsigned int num_threads) override - { - auto matrix_multiply = support::cpp14::make_unique<NEGEMMInterleavedMatrixMultiplyWrapperTemplate<strategy>>(); - matrix_multiply->configure(transformed_a, transformed_b, tmp_c, c, block_walker, block_sizes, params, gemm_info, alpha, beta, num_threads); - return std::move(matrix_multiply); - } - - BlockSizes calculate_block_sizes_for_strategy(const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) override - { - return calculate_block_sizes<strategy>(ci, params.M, params.N, params.K); - } -}; - -/** Create the backend GEMM strategy to use given the provided kernel info - * - * @param[in] kernel_name Kernel name of the backend strategy to instantiate - * - * @return The requested kernel strategy if exists else nullptr - */ -std::unique_ptr<IInterleavedStrategy> create_strategy(const std::string &kernel_name) -{ -#if defined(__arm__) - if(kernel_name.find("sgemm_8x6") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::sgemm_8x6>>(); - } -#endif // defined(__arm__) -#if defined(__aarch64__) - if(kernel_name.find("gemm_s8_4x4") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_s8_4x4>>(); - } - if(kernel_name.find("gemm_s8_12x8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_s8_12x8>>(); - } - if(kernel_name.find("gemm_u8_4x4") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_u8_4x4>>(); - } - if(kernel_name.find("gemm_u8_12x8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::gemm_u8_12x8>>(); - } -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) - if(kernel_name.find("hgemm_24x8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::hgemm_24x8>>(); - } -#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) - if(kernel_name.find("sgemm_12x8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::sgemm_12x8>>(); - } -#if defined(__ARM_FEATURE_SVE) - if(kernel_name.find("interleaved_fp16_mla_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_fp16_mla_3VLx8>>(); - } - if(kernel_name.find("interleaved_fp32_mla_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_fp32_mla_3VLx8>>(); - } - if(kernel_name.find("interleaved_s8s32_dot_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_s8s32_dot_3VLx8>>(); - } - if(kernel_name.find("interleaved_u8u32_dot_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique<InterleavedStrategy<arm_gemm::interleaved_u8u32_dot_3VLx8>>(); - } -#endif // defined(__ARM_FEATURE_SVE) -#endif // defined(__aarch64__)_ - return nullptr; -} -} // namespace detail -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ */ diff --git a/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp b/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp deleted file mode 100644 index ecdb5a938c..0000000000 --- a/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h" - -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/core/WindowIterator.h" - -#include "../arm_gemm/utils.hpp" -#include "arm_gemm.hpp" - -#include "../arm_gemm/mergeresults.hpp" -#include "../arm_gemm/transform.hpp" - -#include "../arm_gemm/kernels/a64_sgemm_native_16x4.hpp" - -namespace arm_compute -{ -namespace -{ -template <typename To, typename Tr> -struct Kernel -{ -}; - -#ifdef __aarch64__ -template <> -struct Kernel<float, float> -{ - using strategy = arm_gemm::sgemm_native_16x4; -}; -#endif /* __aarch64__ */ - -} // namespace - -template <typename To, typename Tr> -Window NEGEMMNativeWrapperKernel<To, Tr>::configure_internal(float alpha, float beta) -{ - ARM_COMPUTE_UNUSED(alpha); - using strategy = typename Kernel<To, Tr>::strategy; - - _beta = beta; - - //Note: The window is shifted down by 1 dimension compare to the tensors - Window window; - window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_params.M, strategy::out_height()), strategy::out_height())); - window.set(Window::DimY, Window::Dimension(0, _params.batches)); - window.set(Window::DimZ, Window::Dimension(0, _params.multis)); - - return window; -} - -template <typename To, typename Tr> -void NEGEMMNativeWrapperKernel<To, Tr>::run_internal(const Window &window, const Coordinates &start_offset, const Coordinates &end_offset, const ThreadInfo &info) -{ - using strategy = typename Kernel<To, Tr>::strategy; - - TensorAccessor<To> a(*_a); - TensorAccessor<To> b(*_b); - TensorAccessor<Tr> c(*_c); - - // Handle 3d input re-interpretation - if(_gemm_info.reinterpret_input_as_3d()) - { - Strides a_strides_as_3d = _a->info()->strides_in_bytes(); - a_strides_as_3d.remove(Window::DimZ); - a.set_strides(a_strides_as_3d); - } - - // Handle 3d output re-interpretation - if(_gemm_info.depth_output_gemm3d() != 0) - { - Strides c_strides_as_3d = _c->info()->strides_in_bytes(); - c_strides_as_3d.remove(Window::DimZ); - c.set_strides(c_strides_as_3d); - } - - unsigned int m_end = 0; - - strategy strat(info.cpu_info); - auto window_iterator = arm_compute::create_window_iterator(window, start_offset, end_offset, [&](const Coordinates & id) - { - const unsigned int y0 = id.x(); - const unsigned int batch = id.y(); - const unsigned int multi = id.z(); - const unsigned int ymax = std::min(y0 + strategy::out_height(), m_end); - - // TODO(COMPMID-1424) : Agree on gemm IO layouts - strat.kernel(a(0, y0, batch, multi), a.stride(Window::DimY), - b(0, 0, multi), b.stride(Window::DimY), - c(0, y0, batch, multi), c.stride(Window::DimY), - _beta, (ymax - y0), _params.N, _params.K); - }); - - auto on_new_row_size = [&](unsigned int start, unsigned int end) - { - ARM_COMPUTE_UNUSED(start); - m_end = std::min(end, _params.M); - }; - - window_iterator.iterate_3D(on_new_row_size); -} - -#ifdef __aarch64__ -template class NEGEMMNativeWrapperKernel<float, float>; -#endif /* __aarch64__ */ - -} // namespace arm_compute |