From 48b3ef89de5f21a0169d8416e3d54081f82c7bf8 Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Mon, 14 Oct 2019 19:03:09 +0100 Subject: COMPMID-2577: Fuse bias addition and activation in gemm assembly kernels Change-Id: I7f52112d2d05b1ea3d3f3d4b19b8eafab05d6c44 Signed-off-by: Georgios Pinitas Reviewed-on: https://review.mlplatform.org/c/2141 Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Pablo Marquez --- arm_compute/core/NEON/kernels/assembly/Helpers.h | 6 +- .../NEGEMMInterleavedMatrixMultiplyWrapper.h | 233 - .../NEGEMMInterleavedPrepareBWrapperKernel.h | 251 - .../assembly/NEGEMMInterleavedTransformAWrapper.h | 173 - .../kernels/assembly/NEGEMMNativeWrapperKernel.h | 52 - .../core/NEON/kernels/assembly/arm_gemm.hpp | 34 +- .../core/NEON/kernels/assembly/gemm_common.hpp | 23 +- arm_compute/runtime/NEON/functions/NEGEMM.h | 36 +- .../NEON/functions/NEGEMMAssemblyDispatch.h | 40 +- .../NEON/functions/NEGEMMConvolutionLayer.h | 6 - .../NEON/functions/NEGEMMLowpMatrixMultiplyCore.h | 4 + .../functions/assembly/NEGEMMInterleavedWrapper.h | 147 - docs/05_functions_list.dox | 1 - src/core/NEON/kernels/NEActivationLayerKernel.cpp | 15 +- src/core/NEON/kernels/arm_gemm/bias_adder.hpp | 75 + src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp | 32 +- src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp | 90 +- src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp | 35 +- .../kernels/arm_gemm/gemm_hybrid_quantized.hpp | 19 +- .../NEON/kernels/arm_gemm/gemm_implementation.hpp | 40 +- src/core/NEON/kernels/arm_gemm/gemm_int16.cpp | 8 +- src/core/NEON/kernels/arm_gemm/gemm_int8.cpp | 46 +- .../NEON/kernels/arm_gemm/gemm_interleaved.hpp | 14 +- src/core/NEON/kernels/arm_gemm/gemm_native.hpp | 21 +- src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp | 32 +- src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp | 8 +- src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp | 46 +- src/core/NEON/kernels/arm_gemm/gemv_batched.hpp | 14 +- .../kernels/arm_gemm/gemv_native_transposed.hpp | 19 +- .../NEON/kernels/arm_gemm/gemv_pretransposed.hpp | 21 +- .../kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp | 4 +- .../kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp | 6 +- .../kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp | 6 +- .../arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp | 27 +- .../kernels/a64_hybrid_fp32_mla_16x4/a55.cpp | 452 +- .../kernels/a64_hybrid_fp32_mla_16x4/generic.cpp | 285 +- .../arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp | 27 +- .../kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp | 2166 ++++----- .../kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp | 2140 ++++---- .../arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp | 25 +- .../kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp | 1852 ++++--- .../kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp | 1852 ++++--- .../arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp | 83 + .../kernels/a64_native_fp32_mla_16x4/generic.cpp | 1708 +++++++ .../a64_sgemm_nativeA_pretransposeB_16x4.hpp | 78 - .../generic.cpp | 970 ---- .../arm_gemm/kernels/a64_sgemm_native_16x4.hpp | 71 - .../kernels/a64_sgemm_native_16x4/generic.cpp | 889 ---- .../kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp | 83 + .../a64_smallK_hybrid_fp32_mla_4x6/generic.cpp | 4612 ++++++++++++++++++ .../kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp | 85 + .../a64_smallK_hybrid_fp32_mla_4x8/generic.cpp | 3340 +++++++++++++ .../kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp | 27 +- .../a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp | 4917 ++++++++++--------- .../a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp | 5113 ++++++++++---------- .../kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp | 27 +- .../a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp | 2611 +++++----- .../a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp | 2581 +++++----- .../kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp | 25 +- .../a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp | 4491 ++++++++--------- .../a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp | 4707 +++++++++--------- .../kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp | 25 +- .../a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp | 2013 ++++---- .../a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp | 2013 ++++---- .../arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp | 30 +- .../kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp | 858 ++-- .../arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp | 30 +- .../kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp | 954 ++-- .../kernels/sve_hybrid_s8s32_dot_4VLx4.hpp | 30 +- .../kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp | 454 +- .../kernels/sve_hybrid_u8u32_dot_4VLx4.hpp | 28 +- .../kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp | 454 +- .../kernels/sve_interleaved_fp16_mla_3VLx8.hpp | 5 +- .../kernels/sve_interleaved_fp32_mla_3VLx8.hpp | 5 +- .../kernels/sve_interleaved_s8s32_dot_3VLx8.hpp | 5 +- .../kernels/sve_interleaved_u8u32_dot_3VLx8.hpp | 7 +- .../arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp | 33 +- .../kernels/sve_native_fp16_mla_4VLx4/generic.cpp | 868 ++-- .../arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp | 32 +- .../kernels/sve_native_fp32_mla_4VLx4/generic.cpp | 977 ++-- .../kernels/sve_native_s8s32_dot_4VLx4.hpp | 31 +- .../kernels/sve_native_s8s32_dot_4VLx4/generic.cpp | 1368 +++--- .../kernels/sve_native_u8u32_dot_4VLx4.hpp | 30 +- .../kernels/sve_native_u8u32_dot_4VLx4/generic.cpp | 1368 +++--- .../arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp | 73 - .../kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp | 4264 ---------------- .../kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp | 73 - .../sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp | 4004 --------------- .../kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp | 24 +- .../sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp | 4 +- .../kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp | 24 +- .../sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp | 4 +- src/core/NEON/kernels/arm_gemm/mergeresults.cpp | 107 + src/core/NEON/kernels/arm_gemm/mergeresults.hpp | 51 +- .../arm_gemm/merges/a32_merge_float_8x6.hpp | 226 +- .../arm_gemm/merges/a64_merge_float_12x8.hpp | 346 -- .../merges/a64_merge_float_to_half_12x8.hpp | 428 -- .../arm_gemm/merges/a64_merge_fp16_24x8.hpp | 2074 ++++++++ .../arm_gemm/merges/a64_merge_fp32_12x8.hpp | 2595 +++++----- .../arm_gemm/merges/a64_merge_half_24x8.hpp | 363 -- .../arm_gemm/merges/a64_merge_int32_12x8.hpp | 348 -- .../kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp | 1595 ++++++ .../kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp | 433 ++ .../kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp | 1595 ++++++ .../kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp | 433 ++ src/core/NEON/kernels/arm_gemm/merges/list.hpp | 11 +- .../arm_gemm/merges/sve_merge_fp16_3VLx8.hpp | 1879 +++++++ .../arm_gemm/merges/sve_merge_fp32_2VLx8.hpp | 1208 ----- .../arm_gemm/merges/sve_merge_fp32_3VLx8.hpp | 2597 +++++----- .../arm_gemm/merges/sve_merge_s32_3VLx8.hpp | 1890 ++++---- .../arm_gemm/merges/sve_merge_u32_3VLx8.hpp | 1890 ++++---- src/core/NEON/kernels/arm_gemm/misc.cpp | 2 +- src/core/NEON/kernels/arm_gemm/profiler.hpp | 137 - .../NEON/kernels/arm_gemm/quantize_wrapper.hpp | 29 +- src/core/NEON/kernels/arm_gemm/quantized.cpp | 23 +- src/core/NEON/kernels/arm_gemm/quantized.hpp | 2 +- .../NEON/kernels/arm_gemm/std_transforms_fixed.hpp | 4 +- .../NEON/kernels/arm_gemm/std_transforms_sve.hpp | 4 +- src/core/NEON/kernels/arm_gemm/transform.hpp | 1 - src/core/NEON/kernels/assembly/Helpers.cpp | 13 +- .../kernels/assembly/NEGEMMInterleavedStrategies.h | 230 - .../kernels/assembly/NEGEMMNativeWrapperKernel.cpp | 130 - src/runtime/NEON/functions/NEGEMM.cpp | 100 +- .../NEON/functions/NEGEMMAssemblyDispatch.cpp | 180 +- .../NEON/functions/NEGEMMConvolutionLayer.cpp | 118 +- .../NEGEMMLowpAssemblyMatrixMultiplyCore.cpp | 2 +- .../functions/NEGEMMLowpMatrixMultiplyCore.cpp | 35 +- .../assembly/NEGEMMInterleavedWrapper.cpp | 430 -- 128 files changed, 46428 insertions(+), 41970 deletions(-) delete mode 100644 arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h delete mode 100644 arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h delete mode 100644 arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h delete mode 100644 arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h delete mode 100644 arm_compute/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.h create mode 100644 src/core/NEON/kernels/arm_gemm/bias_adder.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp create mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp create mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp create mode 100644 src/core/NEON/kernels/arm_gemm/mergeresults.cpp delete mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp create mode 100644 src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp delete mode 100644 src/core/NEON/kernels/arm_gemm/profiler.hpp delete mode 100644 src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h delete mode 100644 src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp delete mode 100644 src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp diff --git a/arm_compute/core/NEON/kernels/assembly/Helpers.h b/arm_compute/core/NEON/kernels/assembly/Helpers.h index e2a46e96a3..092ce400d1 100644 --- a/arm_compute/core/NEON/kernels/assembly/Helpers.h +++ b/arm_compute/core/NEON/kernels/assembly/Helpers.h @@ -47,8 +47,7 @@ struct BlockSizes * @param[in] ci CPU information. * @param[in] num_threads Maximum number of threads that might be used for the calculations. * @param[in] p M, N, K sizes. - * @param[in] alpha Alpha value. - * @param[in] beta Beta value. + * @param[in] activation Activation struct * @param[in] pretranspose_hint Is B also pretransposed ? * * @return Kernel description that the assembly heuristics picked for the given configuration @@ -57,8 +56,7 @@ arm_gemm::KernelDescription get_gemm_info(DataType in const CPUInfo &ci, const unsigned int num_threads, const INEGEMMWrapperKernel::Params &p, - float alpha, - float beta, + arm_gemm::Activation activation, bool pretranspose_hint); /** Calculate the recommended block sizes to use based on the CPU cache sizes and the strategy which will be used diff --git a/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h b/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h deleted file mode 100644 index 641f88ee5f..0000000000 --- a/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMINTERLEAVEDMATRIXMULTIPLYWRAPPER_H__ -#define __ARM_COMPUTE_NEGEMMINTERLEAVEDMATRIXMULTIPLYWRAPPER_H__ - -#include "arm_compute/core/NEON/kernels/assembly/Helpers.h" - -#include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/core/Validate.h" -#include "arm_compute/core/Window.h" -#include "arm_compute/core/WindowIterator.h" - -namespace arm_compute -{ -class ITensor; - -/** Unit of work for @ref NEGEMMInterleavedMatrixMultiplyWrapper to process */ -struct MatrixMultiplyWorkload -{ - /** Constructor - * - * @param[in] offset_transformed_b Offset from the start of transformed_b's allocation. - * @param[in] x0 First value to process along the X dimension (N). - * @param[in] xmax Last value to process along the X dimension (N). - * @param[in] k0 First value to process along the K dimension. - * @param[in] kmax Last value to process along the K dimension. - * @param[in] multi Multi index. - * @param[in] kern_k Number of elements along K actually processed by the kernel. - * @param[in] bblocks Number of x_block processed by the kernel. - */ - MatrixMultiplyWorkload(unsigned int offset_transformed_b, unsigned int x0, unsigned int xmax, unsigned int k0, unsigned int kmax, unsigned int multi, int kern_k, int bblocks) - : _offset_transformed_b(offset_transformed_b), _x0(x0), _xmax(xmax), _k0(k0), _kmax(kmax), _multi(multi), _kern_k(kern_k), _bblocks(bblocks) - { - } - unsigned int _offset_transformed_b; /**< Offset from the start of transformed_b's allocation.*/ - unsigned int _x0; /**< First value to process along the X dimension (N). */ - unsigned int _xmax; /**< Last value to process along the X dimension (N). */ - unsigned int _k0; /**< First value to process along the K dimension. */ - unsigned int _kmax; /**< Last value to process along the K dimension. */ - unsigned int _multi; /**< Multi index. */ - int _kern_k; /**< Number of elements along K actually processed by the kernel. */ - int _bblocks; /**< Number of x_block processed by the kernel. */ -}; - -/** Common interface for the templated wrappers around the matrix multiply NEON assembly implementations */ -class NEGEMMInterleavedMatrixMultiplyWrapper -{ -public: - /** Transform the block at the given coordinates - * - * @param[in] wl Workload to process. - * @param[in] info Information about the current thread. - * @param[in] batch_window Window containing iteration information for the M and batch dimensions. - * @param[in] start_offset Offset relative to the beginning of batch_window to start the processing from. - * @param[in] end_offset Offset relative to the beginning of batch_window to stop the processing. - */ - virtual void transform(const MatrixMultiplyWorkload &wl, const ThreadInfo &info, const Window &batch_window, const Coordinates &start_offset, const Coordinates &end_offset) = 0; - /** Generate an array of workloads - * - * @param[out] workloads Container to store the generated workloads. - */ - virtual void create_workloads(std::vector &workloads) = 0; - /** Default destructor */ - virtual ~NEGEMMInterleavedMatrixMultiplyWrapper() = default; -}; - -/** Equivalent to arm_gemm::GemmInterleaved's strategy::kernel() but using Compute Library types. */ -template -class NEGEMMInterleavedMatrixMultiplyWrapperTemplate : public NEGEMMInterleavedMatrixMultiplyWrapper -{ -public: - /** Configure the matrix multiplication: C = alpha * A * B + beta * C - * - * @param[in] prepared_a Already reshaped matrix A. - * @param[in] transformed_b Already reshaped matrix B. - * @param[out] tmp_c Temporary buffer to be used to store intermediate results. - * @param[in,out] c Result matrix C. - * @param[in] block_walker Window containing iteration information for the M and batch dimensions. - * @param[in] block_sizes Block sizes to use for the matrix multiplication (A & B must have been reshaped using these same block sizes). - * @param[in] params M, N, K sizes. - * @param[in] gemm_info GEMM meta-data - * @param[in] alpha Alpha value - * @param[in] beta Beta value - * @param[in] max_num_threads Maximum number of threads that might be used for the calculations. - */ - void configure(const ITensor *prepared_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, const Window &block_walker, const BlockSizes &block_sizes, - const INEGEMMWrapperKernel::Params ¶ms, const GEMMInfo &gemm_info, float alpha, float beta, unsigned int max_num_threads) - { - _prepared_a = prepared_a; - _transformed_b = transformed_b; - _tmp_c = tmp_c; - _c = c; - _block_walker = block_walker; - _block_sizes = block_sizes; - _params = params; - _b_is_pretransposed = gemm_info.pretranpose_B(); - _reinterpret_c_as_3d = gemm_info.depth_output_gemm3d() != 0; - _alpha = alpha; - _beta = beta; - - auto_init_if_empty(*_tmp_c->info(), c->info()->clone()->set_tensor_shape(TensorShape{ _block_sizes.x_block * strategy::out_height(), max_num_threads })); - } - - // Inherited methods overridden: - void transform(const MatrixMultiplyWorkload &wl, const ThreadInfo &info, const Window &batch_window, const Coordinates &start_offset, const Coordinates &end_offset) override - { - strategy strat(info.cpu_info); - TensorAccessor prepared_a(*_prepared_a); - TensorAccessor transformed_b(*_transformed_b); - TensorAccessor c(*_c); - TensorAccessor tmp_c(*_tmp_c); - - // Handle 3d output re-interpretation - if(_reinterpret_c_as_3d) - { - Strides c_strides_as_3d = _c->info()->strides_in_bytes(); - c_strides_as_3d.remove(Window::DimZ); - c.set_strides(c_strides_as_3d); - } - - int prev_batch = -1; - typename strategy::operand_type *a_ptr = nullptr; - auto window_iterator = arm_compute::create_window_iterator(batch_window, start_offset, end_offset, [&](const Coordinates & id) - { - const unsigned int y = id.x(); - const unsigned int batch = id.y(); - const unsigned int ymax = std::min(_params.M, y + strategy::out_height()); - - // If it's the first block of a new batch then reset the pointer to A. - if(prev_batch != static_cast(batch)) - { - const unsigned int first_m = id.x(); - a_ptr = prepared_a(0, first_m, batch); - prev_batch = batch; - } - - // Call matrix multiply assembly routine to process the block: - strat.kernel(a_ptr, transformed_b(wl._offset_transformed_b), tmp_c(0, info.thread_id), 1, wl._bblocks, wl._kern_k); - a_ptr += strategy::out_height() * wl._kern_k; - - // Merge the result with the other blocks' results: - strat.transforms.Merge(c(0, 0, batch, wl._multi), tmp_c(0, info.thread_id), c.stride(1), y, ymax, wl._x0, wl._xmax, _alpha, (wl._k0 == 0 ? _beta : static_cast(1))); - }); - auto on_new_row_size = [&](unsigned int, unsigned int) - { - //Nothing to do - }; - window_iterator.iterate_2D(on_new_row_size); - } - void create_workloads(std::vector &workloads) override - { - unsigned int offset_transformed_b = 0; - unsigned int wl_index = 0; - unsigned int num_buffers = 0, reshaped_block_size = 0; - - if(!_b_is_pretransposed) - { - num_buffers = _transformed_b->info()->tensor_shape()[1]; - reshaped_block_size = _transformed_b->info()->tensor_shape()[0]; - } - execute_window_loop(_block_walker, [&](const Coordinates & id) - { - const unsigned int x0 = id.x(); - const unsigned int k0 = id.y(); - const unsigned int multi = id.z(); - - const unsigned int xmax = std::min(x0 + _block_walker.x().step(), _params.N); - const unsigned int kmax = std::min(k0 + _block_walker.y().step(), _params.K); - - // Figure out how many "K" the kernel will actually process. - const int kern_k = ceil_to_multiple(kmax - k0, strategy::k_unroll()); - const int bblocks = DIV_CEIL(xmax - x0, strategy::out_width()); - - workloads.push_back(MatrixMultiplyWorkload(offset_transformed_b, x0, xmax, k0, kmax, multi, kern_k, bblocks)); - - if(_b_is_pretransposed) - { - offset_transformed_b += bblocks * strategy::out_width() * kern_k; - } - else - { - // Rotate through the BufferManager's buffers: - wl_index++; - offset_transformed_b = (wl_index % num_buffers) * reshaped_block_size; - } - }); - } - -private: - const ITensor *_prepared_a - { - nullptr - }; - const ITensor *_transformed_b{ nullptr }; - ITensor *_tmp_c{ nullptr }; - ITensor *_c{ nullptr }; - unsigned int _Nsize{ 0 }; - unsigned int _Ksize{ 0 }; - bool _transpose_b{ false }; - BlockSizes _block_sizes{}; - INEGEMMWrapperKernel::Params _params{}; - Window _block_walker{}; - bool _b_is_pretransposed{ false }; - bool _reinterpret_c_as_3d{ false }; - typename strategy::result_type _alpha{}; - typename strategy::result_type _beta{}; -}; -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDMATRIXMULTIPLYWRAPPER_H__ */ diff --git a/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h b/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h deleted file mode 100644 index ba3223f66d..0000000000 --- a/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMINTERLEAVEDPREPAREBWRAPPERKERNEL_H__ -#define __ARM_COMPUTE_NEGEMMINTERLEAVEDPREPAREBWRAPPERKERNEL_H__ - -#include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/NEON/INEKernel.h" -#include "arm_compute/core/NEON/kernels/assembly/Helpers.h" -#include "arm_compute/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/core/Validate.h" - -namespace arm_compute -{ -/** Unit of work for @ref NEGEMMInterleavedPrepareBWrapperKernel to process */ -struct PrepareBWorkload -{ - /** Constructor - * - * @param[in] offset_b Offset from the start of b's allocation - * @param[in] offset_transformed_b Offset from the start of transformed_b's allocation. - * @param[in] x0 First value to process along the X dimension (N). - * @param[in] xmax Last value to process along the X dimension (N). - * @param[in] k0 First value to process along the K dimension. - * @param[in] kmax Last value to process along the K dimension. - */ - PrepareBWorkload(unsigned int offset_b, unsigned int offset_transformed_b, unsigned int x0, unsigned int xmax, unsigned int k0, unsigned int kmax) - : _offset_b(offset_b), _offset_transformed_b(offset_transformed_b), _x0(x0), _xmax(xmax), _k0(k0), _kmax(kmax) - { - } - unsigned int _offset_b; /**< Offset from the start of b's allocation.*/ - unsigned int _offset_transformed_b; /**< Offset from the start of transformed_b's allocation.*/ - unsigned int _x0; /**< First value to process along the X dimension (N). */ - unsigned int _xmax; /**< Last value to process along the X dimension (N). */ - unsigned int _k0; /**< First value to process along the K dimension. */ - unsigned int _kmax; /**< Last value to process along the K dimension. */ -}; - -namespace detail -{ -// Call the lambda function for each workload generated by the passed window. -template -void for_each_element_in_window(const Window &window, const ITensor *b, ITensor *transformed_b, unsigned int N, unsigned int K, Lambda &&lambda) -{ - unsigned int wl_index = 0; - unsigned int num_buffers = 0, reshaped_block_size = 0; - - if(use_buffer_manager) - { - num_buffers = transformed_b->info()->tensor_shape()[1]; - reshaped_block_size = transformed_b->info()->strides_in_bytes().y(); - } - - unsigned int offset_transformed_b = transformed_b->info()->offset_first_element_in_bytes(); - execute_window_loop(window, [&](const Coordinates & coordinates) - { - const unsigned int x0 = coordinates.x(); - const unsigned int k0 = coordinates.y(); - const unsigned int multi = coordinates.z(); - - const unsigned int offset_b = b->info()->offset_element_in_bytes(Coordinates(0, 0, multi)); - const unsigned int xmax = std::min(x0 + window.x().step(), N); - const unsigned int kmax = std::min(k0 + window.y().step(), K); - - /* Figure out the size of each block. */ - unsigned int x_size = (xmax - x0); - unsigned int k_size = (kmax - k0); - - /* Round sizes up as needed. */ - x_size = ceil_to_multiple(x_size, strategy::out_width()); - k_size = ceil_to_multiple(k_size, strategy::k_unroll()); - - lambda(PrepareBWorkload(offset_b, offset_transformed_b, x0, xmax, k0, kmax)); - - //Each workload represents one block: - if(use_buffer_manager) - { - // Rotate through the BufferManager's buffers: - wl_index++; - offset_transformed_b = (wl_index % num_buffers) * reshaped_block_size; - } - else - { - offset_transformed_b += (x_size * k_size * sizeof(typename strategy::operand_type)); - } - }); -} - -// Calculate the size of transformed_b: -template -unsigned int get_B_pretransposed_array_size(unsigned int N, unsigned int K, const BlockSizes &bs, unsigned int multis) -{ - // How many full blocks do N / K contain ? - size_t num_full_k = K / bs.k_block; - size_t num_full_x = N / bs.x_block; - - ARM_COMPUTE_ERROR_ON(bs.x_block % strategy::out_width() != 0); - ARM_COMPUTE_ERROR_ON(bs.k_block % strategy::k_unroll() != 0); - - size_t normal_x_size = bs.x_block; - size_t normal_k_size = bs.k_block; - - // Round up the leftovers to be a multiple of the strategy processing size: - size_t left_over_x_size = ceil_to_multiple(N % bs.x_block, strategy::out_width()); - size_t left_over_k_size = ceil_to_multiple(K % bs.k_block, strategy::k_unroll()); - - // Calculate the total size of the buffer: - size_t total = num_full_k * normal_k_size * (num_full_x * normal_x_size + left_over_x_size); - total += left_over_k_size * (left_over_x_size + num_full_x * normal_x_size); - - total *= multis; - - return total; -} -} // namespace detail - -/** Common interface for the templated wrappers around the B reshape NEON assembly implementations */ -class NEGEMMInterleavedPrepareBWrapperKernel : public INEKernel -{ -public: - /** Transform the block at the given coordinates - * - * @param[in] wl Workload to process. - * @param[in] info Information about the current thread. - */ - virtual void transform(const PrepareBWorkload &wl, const ThreadInfo &info) = 0; - /** Generate an array of workloads - * - * @param[out] workloads Container to store the generated workloads. - */ - virtual void create_workloads(std::vector &workloads) = 0; - /** Return the block_sizes used to resape B - * - * The same block sizes must be used to reshape A and for the matrix multiplication - * - * @return The block sizes used to reshape B. - */ - virtual BlockSizes block_sizes() const = 0; - - // Inherited methods overridden: - const char *name() const override - { - return "NEGEMMInterleavedPrepareBWrapperKernel"; - } - - bool is_parallelisable() const override - { - return false; // Can't run on arbitrary windows but can be parallelised using an array of workloads - } -}; - -/** Equivalent to arm_gemm::GemmInterleaved's strategy::transforms::PrepareB() but using Compute Library types. - */ -template -class NEGEMMInterleavedPrepareBWrapperKernelTemplate : public NEGEMMInterleavedPrepareBWrapperKernel -{ -public: - /** Configure the reshape B routine. - * - * @param[in] b Input matrix B. - * @param[out] transformed_b Reshaped matrix B. - * @param[in] transpose_b Also transpose B ? - * @param[in] ci CPU information - * @param[in] params M, N, K sizes. - */ - void configure(const ITensor *b, ITensor *transformed_b, bool transpose_b, const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) - { - const unsigned int multis = b->info()->tensor_shape().z(); - _Nsize = b->info()->tensor_shape().x(); - _Ksize = b->info()->tensor_shape().y(); - _b = b; - _transformed_b = transformed_b; - _transpose_b = transpose_b; - - _block_sizes = calculate_block_sizes(ci, params.M, params.N, params.K); - - auto_init_if_empty(*transformed_b->info(), b->info()->clone()->set_tensor_shape(TensorShape{ detail::get_B_pretransposed_array_size(_Nsize, _Ksize, _block_sizes, multis) })); - - Window window; - window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_Nsize, _block_sizes.x_block), _block_sizes.x_block)); - window.set(Window::DimY, Window::Dimension(0, ceil_to_multiple(_Ksize, _block_sizes.k_block), _block_sizes.k_block)); - window.set(Window::DimZ, Window::Dimension(0, multis)); - - INEKernel::configure(window); - } - - // Inherited methods overridden: - void transform(const PrepareBWorkload &wl, const ThreadInfo &info) override - { - strategy strat(info.cpu_info); - strat.transforms.PrepareB(reinterpret_cast(_transformed_b->buffer() + wl._offset_transformed_b), - reinterpret_cast(_b->buffer() + wl._offset_b), - _b->info()->strides_in_bytes().y() / sizeof(typename strategy::operand_type), - wl._x0, wl._xmax, wl._k0, wl._kmax, _transpose_b); - } - void create_workloads(std::vector &workloads) override - { - detail::for_each_element_in_window(window(), _b, _transformed_b, _Nsize, _Ksize, [&workloads](PrepareBWorkload && wl) - { - workloads.push_back(std::move(wl)); - }); - } - void run(const Window &window, const ThreadInfo &info) override - { - ARM_COMPUTE_ERROR_ON_MISMATCHING_WINDOWS(window, INEKernel::window()); - detail::for_each_element_in_window(window, _b, _transformed_b, _Nsize, _Ksize, [&](PrepareBWorkload && wl) - { - this->transform(wl, info); - }); - } - BlockSizes block_sizes() const override - { - return _block_sizes; - } - -private: - const ITensor *_b - { - nullptr - }; - ITensor *_transformed_b{ nullptr }; - unsigned int _Nsize{ 0 }; - unsigned int _Ksize{ 0 }; - bool _transpose_b{ false }; - BlockSizes _block_sizes{}; -}; - -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDPREPAREBWRAPPERKERNEL_H__ */ diff --git a/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h b/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h deleted file mode 100644 index c1fd86e453..0000000000 --- a/arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMINTERLEAVEDTRANSFORMAWRAPPER_H__ -#define __ARM_COMPUTE_NEGEMMINTERLEAVEDTRANSFORMAWRAPPER_H__ - -#include "arm_compute/core/CPP/CPPTypes.h" -#include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/core/Validate.h" -#include "arm_compute/core/Window.h" -#include "arm_compute/core/WindowIterator.h" - -namespace arm_compute -{ -class ITensor; - -/** Unit of work for @ref NEGEMMInterleavedTransformAWrapper to process */ -struct TransformAWorkload -{ - /** Constructor - * - * @param[in] k0 First value to process along the K dimension. - * @param[in] kmax Last value to process along the K dimension. - * @param[in] multi Multi index. - */ - TransformAWorkload(unsigned int k0, unsigned int kmax, unsigned int multi) - : _k0(k0), _kmax(kmax), _multi(multi) - { - } - unsigned int _k0; /**< First value to process along the K dimension. */ - unsigned int _kmax; /**< Last value to process along the K dimension. */ - unsigned int _multi; /**< Multi index. */ -}; - -/** Equivalent to arm_gemm::GemmInterleaved's Transform &workloads) = 0; - /** Default destructor */ - virtual ~NEGEMMInterleavedTransformAWrapper() = default; -}; - -/** Type specialisations of @ref NEGEMMInterleavedTransformAWrapper */ -template -class NEGEMMInterleavedTransformAWrapperTemplate : public NEGEMMInterleavedTransformAWrapper -{ -public: - /** Configure the reshape A routine. - * - * @param[in] a Input matrix A. - * @param[out] transformed_a Reshaped matrix A. - * @param[in] transpose_a Also transpose A ? - * @param[in] reinterpret_a_as_3d Re-interpret as 3D ? - * @param[in] block_walker Window representing the layout of the matrix's blocks - * @param[in] params M, N, K sizes. - */ - void configure(const ITensor *a, ITensor *transformed_a, bool transpose_a, bool reinterpret_a_as_3d, const Window &block_walker, const INEGEMMWrapperKernel::Params ¶ms) - { - _a = a; - _transformed_a = transformed_a; - _transpose_a = transpose_a; - _reinterpret_a_as_3d = reinterpret_a_as_3d; - _Ksize = params.K; - _Msize = params.M; - _k_multi_window = block_walker.shift_dimensions(1); // block_walker contains (M,K,Multi) --> shift by 1 to get rid of the "M" dimension - } - - // Inherited methods overridden: - void transform(const TransformAWorkload &wl, const ThreadInfo &info, const Window &batch_window, const Coordinates &start_offset, const Coordinates &end_offset) override - { - strategy strat(info.cpu_info); - TensorAccessor a(*_a); - TensorAccessor transformed_a(*_transformed_a); - - // Handle 3d input re-interpretation - if(_reinterpret_a_as_3d) - { - Strides a_strides_as_3d = _a->info()->strides_in_bytes(); - a_strides_as_3d.remove(Window::DimZ); - a.set_strides(a_strides_as_3d); - } - - unsigned int last_m = 0; - //TODO: Create a new iterate_1D( DimY); - int last_y = -1; - auto window_iterator = arm_compute::create_window_iterator(batch_window, start_offset, end_offset, [&](const Coordinates & id) - { - if(id.y() != last_y) - { - last_y = id.y(); - unsigned int batch = id.y(); - unsigned int first_m = id.x(); - - if(first_m >= last_m) - return; - - strat.transforms.PrepareA(transformed_a(0, first_m, batch), - a(0, 0, batch, wl._multi), - a.stride(1), first_m, last_m, wl._k0, wl._kmax, _transpose_a); - } - }); - auto on_new_row_size = [&](unsigned int, unsigned int end) - { - last_m = std::min(end, _Msize); - }; - window_iterator.iterate_2D(on_new_row_size); - } - void create_workloads(std::vector &workloads) override - { - execute_window_loop(_k_multi_window, [&](const Coordinates & id) - { - const unsigned int k0 = id.x(); - const unsigned int multi = id.y(); - const unsigned int kmax = std::min(k0 + _k_multi_window.x().step(), _Ksize); - - workloads.push_back(TransformAWorkload(k0, kmax, multi)); - }); - } - -private: - const ITensor *_a - { - nullptr - }; - ITensor *_transformed_a{ nullptr }; - unsigned int _Msize{ 0 }; - unsigned int _Ksize{ 0 }; - bool _transpose_a{ false }; - bool _reinterpret_a_as_3d{ false }; - Window _k_multi_window{}; -}; -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDTRANSFORMAWRAPPER_H__ */ diff --git a/arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h b/arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h deleted file mode 100644 index 73a0d7f05f..0000000000 --- a/arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMNATIVEWRAPPERKERNEL_H__ -#define __ARM_COMPUTE_NEGEMMNATIVEWRAPPERKERNEL_H__ - -#include "INEGEMMWrapperKernel.h" - -namespace arm_compute -{ -/** Equivalent to arm_gemm::GemmNative but using Compute Library types. - */ -template -class NEGEMMNativeWrapperKernel : public INEGEMMWrapperKernel -{ -public: - const char *name() const override - { - return "NEGEMMNativeWrapperKernel"; - } - -protected: - // Inherited methods overridden: - Window configure_internal(float alpha, float beta) override; - void run_internal(const Window &window, const Coordinates &start_offset, const Coordinates &end_offset, const ThreadInfo &info) override; - -private: - Tr _beta{}; -}; - -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMNATIVEWRAPPERKERNEL_H__ */ diff --git a/arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp b/arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp index 828b0f20a7..17faab18fd 100644 --- a/arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp +++ b/arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp @@ -65,7 +65,21 @@ struct GemmConfig GemmConfig() { } }; -template +struct Activation +{ + enum class Type { + None, + ReLU, + BoundedReLU + }; + + Type type; + float param1; + float param2; + + Activation(Type type=Type::None, float p1=0.0f, float p2=0.0f) : type(type), param1(p1), param2(p2) { } +}; + struct GemmArgs { public: @@ -77,8 +91,7 @@ public: unsigned int _nmulti; bool _trA; bool _trB; - T _alpha; - T _beta; + Activation _act; int _maxthreads; bool _pretransposed_hint; const GemmConfig *_cfg; @@ -86,10 +99,10 @@ public: GemmArgs(const CPUInfo *ci, const unsigned int M, const unsigned int N, const unsigned int K, const unsigned int nbatches, const unsigned int nmulti, const bool trA, const bool trB, - const T alpha, const T beta, const int maxthreads, + Activation act, const int maxthreads, const bool pretransposed_hint, const GemmConfig *cfg=nullptr ) : _ci(ci), _Msize(M), _Nsize(N), _Ksize(K), _nbatches(nbatches), _nmulti(nmulti), - _trA(trA), _trB(trB), _alpha(alpha), _beta(beta), _maxthreads(maxthreads), + _trA(trA), _trB(trB), _act(act), _maxthreads(maxthreads), _pretransposed_hint(pretransposed_hint), _cfg(cfg) { } @@ -99,6 +112,7 @@ struct ARequantizeLayer32 { public: const int32_t *bias; + size_t bias_multi_stride; int32_t a_offset; int32_t b_offset; int32_t c_offset; @@ -109,8 +123,8 @@ public: ARequantizeLayer32() = default; - ARequantizeLayer32(int32_t *b, int32_t ao, int32_t bo, int32_t co, int32_t rs, int32_t rm, int32_t minv, int32_t maxv) : - bias(b), a_offset(ao), b_offset(bo), c_offset(co), requant_shift(rs), requant_mul(rm), minval(minv), maxval(maxv) + ARequantizeLayer32(const int32_t *b, size_t bms, int32_t ao, int32_t bo, int32_t co, int32_t rs, int32_t rm, int32_t minv, int32_t maxv) : + bias(b), bias_multi_stride(bms), a_offset(ao), b_offset(bo), c_offset(co), requant_shift(rs), requant_mul(rm), minval(minv), maxval(maxv) { } }; @@ -128,12 +142,12 @@ using UniqueGemmCommon = std::unique_ptr >; /* get_gemm_method(): Given the templated types and provided parameters, * which is the preferred method to implement this GEMM? */ template -KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage & ={}); +KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage & ={}); template -UniqueGemmCommon gemm(const GemmArgs &args, const OutputStage & ={}); +UniqueGemmCommon gemm(const GemmArgs &args, const OutputStage & ={}); template -std::vector get_compatible_kernels(const GemmArgs &args, const OutputStage & ={}); +std::vector get_compatible_kernels(const GemmArgs &args, const OutputStage & ={}); } // namespace arm_gemm diff --git a/arm_compute/core/NEON/kernels/assembly/gemm_common.hpp b/arm_compute/core/NEON/kernels/assembly/gemm_common.hpp index 1ae503cddb..d17fd5fe97 100644 --- a/arm_compute/core/NEON/kernels/assembly/gemm_common.hpp +++ b/arm_compute/core/NEON/kernels/assembly/gemm_common.hpp @@ -48,7 +48,8 @@ public: */ virtual void set_arrays_generic(const void *A, const int lda, const int A_batch_stride, const int A_multi_stride, const void *B, const int ldb, /* batches share B */ const int B_multi_stride, - void *C, const int ldc, const int C_batch_stride, const int C_multi_stride) = 0; + void *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const void *bias, /* no row or batch stride needed */ const int bias_multi_stride) = 0; /* For threading, we divide the work into some number of units and work * out internally what unit corresponds to what work. This returns the @@ -97,7 +98,11 @@ public: /*** "Quantized bias" interface (optional) ***/ /* Set the bias vector for quantized GEMMs */ - virtual void set_quantized_bias(const int32_t *bias) { UNUSED(bias); } + virtual void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) + { + UNUSED(bias); + UNUSED(bias_multi_stride); + } // Destructor virtual ~IGemmCommon() { } @@ -125,13 +130,16 @@ protected: int _ldc=0; int _C_batch_stride=0; int _C_multi_stride=0; + const Tr *_bias=nullptr; + int _bias_multi_stride=0; public: /* Pass in the pointers to the arrays to be operated on and their * strides (templated version with appropriate types). */ virtual void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, /* batches share B */ const int B_multi_stride, - Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) { + Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const Tr *bias, /* no row or batch stride needed */ const int bias_multi_stride) { _Aptr = A; _lda = lda; _A_batch_stride = A_batch_stride; @@ -143,15 +151,19 @@ public: _ldc = ldc; _C_batch_stride = C_batch_stride; _C_multi_stride = C_multi_stride; + _bias = bias; + _bias_multi_stride = bias_multi_stride; } /* Implementation of the void * overload which casts its arguments to the appropriate type. */ void set_arrays_generic(const void *A, const int lda, const int A_batch_stride, const int A_multi_stride, const void *B, const int ldb, /* batches share B */ const int B_multi_stride, - void *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override { + void *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const void *bias, /* no row or batch stride needed */ const int bias_multi_stride) override { set_arrays(static_cast(A), lda, A_batch_stride, A_multi_stride, static_cast(B), ldb, B_multi_stride, - static_cast(C), ldc, C_batch_stride, C_multi_stride); + static_cast(C), ldc, C_batch_stride, C_multi_stride, + static_cast(bias), bias_multi_stride); } /*** "Pretransposed" interface ***/ @@ -164,7 +176,6 @@ public: void pretranspose_B_array_generic(void *out, const void *in, const int row_stride, const int multi_stride) override { pretranspose_B_array(out, static_cast(in), row_stride, multi_stride); } - }; } // namespace arm_gemm diff --git a/arm_compute/runtime/NEON/functions/NEGEMM.h b/arm_compute/runtime/NEON/functions/NEGEMM.h index d947be1ef9..e4d69eb93d 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMM.h +++ b/arm_compute/runtime/NEON/functions/NEGEMM.h @@ -24,6 +24,7 @@ #ifndef __ARM_COMPUTE_NEGEMM_H__ #define __ARM_COMPUTE_NEGEMM_H__ +#include "arm_compute/core/NEON/kernels/NEArithmeticAdditionKernel.h" #include "arm_compute/core/NEON/kernels/NEFillBorderKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMInterleave4x4Kernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMMatrixAdditionKernel.h" @@ -33,20 +34,27 @@ #include "arm_compute/runtime/IMemoryManager.h" #include "arm_compute/runtime/IWeightsManager.h" #include "arm_compute/runtime/MemoryGroup.h" +#include "arm_compute/runtime/NEON/functions/NEActivationLayer.h" #include "arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h" #include "arm_compute/runtime/Tensor.h" -#include - namespace arm_compute { /** Basic function to execute GEMM on NEON. This function calls the following NEON kernels: * + * If optimized assembly is available: + * -# @ref NEGEMMAssemblyDispatch + * -# @ref NEActivationLayer (if alpha != 1.0) + * Else: * -# @ref NEGEMMInterleave4x4Kernel (if the output tensor is a matrix) * -# @ref NEGEMMTranspose1xWKernel (if the output tensor is a matrix) * -# @ref NEGEMMMatrixMultiplyKernel - * -# @ref NEGEMMMatrixAdditionKernel (if c != nullptr and beta != 0.0) + * In both cases: + * -# @ref NEGEMMMatrixAdditionKernel (if c != nullptr and beta != 0.0 and is not reshaped once) + * Else: + * -# @ref NEArithmeticAdditionKernel (if c != nullptr and is reshaped once and not optimized assembly in place) * + * -# @ref NEActivationLayer (if activation is specified in GEMMInfo) */ class NEGEMM : public IFunction { @@ -103,13 +111,21 @@ private: NEGEMMMatrixMultiplyKernel _mm_kernel; NEGEMMAssemblyDispatch _asm_glue; NEGEMMMatrixAdditionKernel _ma_kernel; - Tensor _tmp_a; - Tensor _tmp_b; - const ITensor *_original_b; - bool _run_vector_matrix_multiplication; - bool _run_addition; - bool _reshape_b_only_on_first_run; - bool _is_prepared; + NEActivationLayer _alpha_scale_func; + NEArithmeticAdditionKernel _add_bias_kernel; + NEActivationLayer _activation_func; + + Tensor _tmp_a; + Tensor _tmp_b; + Tensor _tmp_d; + const ITensor *_original_b; + bool _run_vector_matrix_multiplication; + bool _run_alpha_scale; + bool _run_addition; + bool _run_bias_addition; + bool _run_activation; + bool _reshape_b_only_on_first_run; + bool _is_prepared; }; } // namespace arm_compute #endif /*__ARM_COMPUTE_NEGEMM_H__ */ diff --git a/arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h b/arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h index 83e495e695..20d189e76b 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h +++ b/arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h @@ -59,29 +59,10 @@ public: }; private: - /** ACL Function */ - std::unique_ptr _function; - - /** If supported create the ACL function corresponding to the GemmMethod provided to process the other passed parameters - * - * @param[in] method GemmMethod to use to perform the matrix multiplication. - * @param[in] a Input tensor (Matrix A). - * @param[in] b Input tensor (Matrix B). - * @param[in] c Input tensor (Matrix C) used to pass the bias for quantized calculations - * @param[out] d Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0. - * @param[in] alpha Scalar multiplier to apply to AB matrix product. - * @param[in] beta Scalar multiplier to apply to input D matrix before adding product. - * @param[in] gemm_info GEMM meta-data - * - * @return True if the method is supported and the function was successfully created, false otherwise. - */ - bool create_function(arm_gemm::GemmMethod method, const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info); - /** Interface for the arm_gemm fallback */ - std::unique_ptr _arm_gemm; - MemoryGroup _memory_group; /**< Function memory group */ - std::shared_ptr _memory_manager; /**< Copy of the memory manager used to create the memory group to be used when instantiating new functions */ - IWeightsManager *_weights_manager; /**< Pointer to the weights manager */ + std::unique_ptr _arm_gemm; + MemoryGroup _memory_group; /**< Function memory group */ + IWeightsManager *_weights_manager; /**< Pointer to the weights manager */ public: /** If supported create an ACL function else fallback to the arm_gemm function. * @@ -89,11 +70,9 @@ public: * @param[in] b Input tensor (Matrix B) * @param[in] c Input tensor (Matrix C) used to pass the bias for quantized calculations * @param[out] d Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0. - * @param[in] alpha Scalar multiplier to apply to AB matrix product. - * @param[in] beta Scalar multiplier to apply to input D matrix before adding product. * @param[in] gemm_info GEMM meta-data */ - void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info); + void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, const GEMMInfo &gemm_info); /** Indicates whether or not this function can be used to process the given parameters. * @@ -101,13 +80,18 @@ public: * @param[in] b Input tensor info (Matrix B) * @param[in] c Input tensor info (Matrix C) used to pass the bias for quantized calculations * @param[in] d Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0. - * @param[in] alpha Scalar multiplier to apply to AB matrix product. - * @param[in] beta Scalar multiplier to apply to input D matrix before adding product. * @param[in] gemm_info GEMM meta-data * * @return a status. */ - static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, float alpha, float beta, const GEMMInfo &gemm_info); + static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, const GEMMInfo &gemm_info); + /** Checks if activation is supported by the gemm assembly dispatcher + * + * @param[in] activation Activation to check + * + * @return True if activation is supported else false + */ + static bool is_activation_supported(const ActivationLayerInfo &activation); /** Was the function successfully configured ? * * @return True if the function is configured and ready to run diff --git a/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h b/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h index dccc35f0af..3e551abf5a 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h +++ b/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h @@ -26,7 +26,6 @@ #include "arm_compute/runtime/IFunction.h" -#include "arm_compute/core/NEON/kernels/NEArithmeticAdditionKernel.h" #include "arm_compute/core/NEON/kernels/NECol2ImKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMTranspose1xWKernel.h" #include "arm_compute/core/NEON/kernels/NEIm2ColKernel.h" @@ -34,7 +33,6 @@ #include "arm_compute/core/Types.h" #include "arm_compute/runtime/IWeightsManager.h" #include "arm_compute/runtime/MemoryGroup.h" -#include "arm_compute/runtime/NEON/functions/NEActivationLayer.h" #include "arm_compute/runtime/NEON/functions/NEGEMM.h" #include "arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h" #include "arm_compute/runtime/NEON/functions/NEGEMMLowpOutputStage.h" @@ -250,8 +248,6 @@ private: NEGEMM _mm_gemm; NEGEMMLowpMatrixMultiplyCore _mm_gemmlowp; NECol2ImKernel _col2im_kernel; - NEActivationLayer _activationlayer_function; - NEArithmeticAdditionKernel _add_bias_kernel; NEReshapeLayer _reshape_layer; const ITensor *_original_weights; @@ -263,11 +259,9 @@ private: DataLayout _data_layout; - bool _append_bias; bool _skip_im2col; bool _skip_col2im; bool _is_quantized; - bool _is_activationlayer_enabled; bool _is_prepared; }; } // namespace arm_compute diff --git a/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h b/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h index 5b6a0dd943..12c120934e 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h +++ b/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h @@ -24,6 +24,7 @@ #ifndef __ARM_COMPUTE_NEGEMMLOWPMATRIXMULTIPLYCORE_H__ #define __ARM_COMPUTE_NEGEMMLOWPMATRIXMULTIPLYCORE_H__ +#include "NEActivationLayer.h" #include "arm_compute/core/NEON/INEKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpOffsetContributionKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpOffsetContributionOutputStageKernel.h" @@ -46,6 +47,7 @@ class ITensor; * -# @ref NEGEMMTranspose1xWKernel * -# @ref NEGEMMLowpMatrixMultiplyKernel * -# @ref NEGEMMLowpOffsetContributionKernel + * -# @ref NEActivationLayer * * otherwise if the DOT product instruction is available: * @@ -113,6 +115,7 @@ private: NEGEMMLowpMatrixBReductionKernel _mtx_b_reduction_kernel; NEGEMMLowpOffsetContributionKernel _offset_contribution_kernel; NEGEMMLowpOffsetContributionOutputStageKernel _offset_contribution_output_stage_kernel; + NEActivationLayer _activation_func; Tensor _vector_sum_col; Tensor _vector_sum_row; Tensor _tmp_a; @@ -127,6 +130,7 @@ private: bool _reshape_b_only_on_first_run; bool _is_prepared; bool _fuse_output_stage; + bool _run_activation; }; } // namespace arm_compute #endif /*__ARM_COMPUTE_NEGEMMLOWPMATRIXMULTIPLYCORE_H__ */ diff --git a/arm_compute/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.h b/arm_compute/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.h deleted file mode 100644 index 695dcd5b6e..0000000000 --- a/arm_compute/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMINTERLEAVEDWRAPPER_H__ -#define __ARM_COMPUTE_NEGEMMINTERLEAVEDWRAPPER_H__ - -#include "arm_compute/core/NEON/kernels/assembly/Helpers.h" -#include "arm_compute/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h" -#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedMatrixMultiplyWrapper.h" -#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedPrepareBWrapperKernel.h" -#include "arm_compute/core/NEON/kernels/assembly/NEGEMMInterleavedTransformAWrapper.h" -#include "arm_compute/runtime/IFunction.h" -#include "arm_compute/runtime/IMemoryManager.h" -#include "arm_compute/runtime/IScheduler.h" -#include "arm_compute/runtime/IWeightsManager.h" -#include "arm_compute/runtime/MemoryGroup.h" -#include "arm_compute/runtime/Tensor.h" - -#include - -namespace arm_compute -{ -// Forward declarations -class ITensor; - -/** Buffer manager used when reshaping B on the fly - * - * The typical workflow is: - * - lock_to_reshape_if_needed() - * - If the previous lock was successful: mark_as_reshaped() - * - wait_for_reshaping() wait for the reshaping to be complete - * - mark_as_unused() once the thread is done using this given buffer. - * - * Calls for different indices might be interleaved, however the calls for a given index must always be in that order. - */ -class IBufferManager -{ -public: - /** Lock a buffer for the given index if it's available else return - * - * @param[in] index Index of the buffer to lock - * - * @return True if the buffer has been successfully locked, false if it's already reshaped / being reshaped. - */ - virtual bool lock_to_reshape_if_needed(unsigned int index) = 0; - /** Mark a buffer previously locked as reshaped - * - * @pre The thread calling this function must have locked the given buffer through lock_to_reshape_if_needed() - * - * @param[in] index Index of the buffer to mark as reshaped - */ - virtual void mark_as_reshaped(unsigned int index) = 0; - /** Block until the given buffer is marked as reshaped - * - * @param[in] index Index of the buffer - */ - virtual void wait_for_reshaping(unsigned int index) = 0; - /** Mark a reshaped buffer as unused - * - * Once all the users have marked a buffer as unused then it goes back to being free - */ - virtual void mark_as_unused(unsigned int index) = 0; - - /** Number of buffers used internally - * - * @return The number of buffers used by the manager. - */ - virtual unsigned int num_buffers() const = 0; - /** Default destructor */ - virtual ~IBufferManager() = default; -}; - -/** Equivalent to arm_gemm::GemmInterleaved but using Compute Library types. - */ -class NEGEMMInterleavedWrapper : public IFunction -{ -public: - NEGEMMInterleavedWrapper(std::shared_ptr memory_manager = nullptr, IWeightsManager *weights_manager = nullptr); - ~NEGEMMInterleavedWrapper() = default; - - NEGEMMInterleavedWrapper(const NEGEMMInterleavedWrapper &) = delete; - NEGEMMInterleavedWrapper &operator=(const NEGEMMInterleavedWrapper &) = delete; - - /** Initialise the kernel's input and output. - * - * @note The input and output tensor must have the same dimensions - * - * @param[in] a Input tensor (Matrix A) - * @param[in] b Input tensor (Matrix B) - * @param[out] c Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0. - * @param[in] alpha Scalar multiplier to apply to AB matrix product. - * @param[in] beta Scalar multiplier to apply to input C matrix before adding product. - * @param[in] gemm_info GEMM meta-data - */ - void configure(const ITensor *a, const ITensor *b, ITensor *c, float alpha, float beta, const GEMMInfo &gemm_info); - - // Inherited methods overridden: - void run() override; - void prepare() override; - -private: - MemoryGroup _memory_group; - IWeightsManager *_weights_manager; - bool _is_prepared{ false }; - bool _pretranspose_b{ false }; - Window _block_walker{}; - Window _batch_window{}; - const ITensor *_a{ nullptr }; - const ITensor *_b{ nullptr }; - ITensor *_c{ nullptr }; - Tensor _transformed_b{}; - Tensor _transformed_a{}; - Tensor _tmp_c{}; - INEGEMMWrapperKernel::Params _params{}; - BlockSizes _block_sizes{}; - std::unique_ptr _prepare_b{ nullptr }; - std::unique_ptr _transform_a{ nullptr }; - std::unique_ptr _matrix_multiply{ nullptr }; - std::unique_ptr _buffer_manager{ nullptr }; - std::vector _a_workloads{}; - std::vector _b_workloads{}; - std::vector _mm_workloads{}; - std::vector _workloads{}; - std::string _tag{}; -}; -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDWRAPPER_H__ */ diff --git a/docs/05_functions_list.dox b/docs/05_functions_list.dox index 7641732cd6..af7decfb74 100644 --- a/docs/05_functions_list.dox +++ b/docs/05_functions_list.dox @@ -136,7 +136,6 @@ namespace arm_compute - @ref NEGEMM - @ref NEGEMMAssemblyDispatch - @ref NEGEMMConvolutionLayer - - @ref NEGEMMInterleavedWrapper - @ref NEGEMMLowpAssemblyMatrixMultiplyCore - @ref NEGEMMLowpMatrixMultiplyCore - @ref NEHarrisCorners diff --git a/src/core/NEON/kernels/NEActivationLayerKernel.cpp b/src/core/NEON/kernels/NEActivationLayerKernel.cpp index 6f722e0457..c338ef09c7 100644 --- a/src/core/NEON/kernels/NEActivationLayerKernel.cpp +++ b/src/core/NEON/kernels/NEActivationLayerKernel.cpp @@ -109,7 +109,7 @@ std::pair validate_and_configure_window(ITensorInfo *input, ITen } // namespace NEActivationLayerKernel::NEActivationLayerKernel() - : _input(nullptr), _output(nullptr), _func(nullptr), _act_info(ActivationFunction::LOGISTIC) + : _input(nullptr), _output(nullptr), _func(nullptr), _act_info() { } @@ -121,11 +121,18 @@ void NEActivationLayerKernel::configure(ITensor *input, ITensor *output, Activat _act_info = activation_info; _output = input; + // Out-of-place calculation if(output != nullptr) { _output = output; } + // Disabled activation, thus no operation needed + if(!activation_info.enabled()) + { + _func = nullptr; + } + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), (output != nullptr) ? output->info() : nullptr, activation_info)); // Activation functions : FP32 @@ -610,6 +617,12 @@ Status NEActivationLayerKernel::validate(const ITensorInfo *input, const ITensor void NEActivationLayerKernel::run(const Window &window, const ThreadInfo &info) { + // Early exit on disabled activation + if(!_act_info.enabled()) + { + return; + } + ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window); diff --git a/src/core/NEON/kernels/arm_gemm/bias_adder.hpp b/src/core/NEON/kernels/arm_gemm/bias_adder.hpp new file mode 100644 index 0000000000..745d00563b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/bias_adder.hpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +namespace arm_gemm +{ + +// Fallback routine to add bias to a block +template +inline void bias_adder(T *out, unsigned int stride, const T *bias, unsigned int rows, unsigned int cols) { + for (unsigned int row=0; row +inline void activator(T *out, unsigned int stride, const T *bias, Activation act, unsigned int rows, unsigned int cols) { + if (act.type == Activation::Type::None) { + if (DoBias) { + bias_adder(out, stride, bias, rows, cols); + } + return; + } + + if (act.type == Activation::Type::ReLU) { + for (unsigned int row=0; row(0), v); + } + } + } + + if (act.type == Activation::Type::BoundedReLU) { + const T max = static_cast(act.param1); + + for (unsigned int row=0; row(0), std::min(v, max)); + } + } + } +} + +} // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp index 8541d34de5..b6671e8c85 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp @@ -47,23 +47,23 @@ static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = { { GemmMethod::GEMM_HYBRID, "hybrid_fp16_mla_4VLx4", - [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<__fp16> &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return (args._Ksize >= 8) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_NATIVE, "native_fp16_mla_4VLx4", - [](const GemmArgs<__fp16> &args) { return (args._Ksize >= 8 && args._alpha==1.0f && !args._trA && !args._trB); }, - [](const GemmArgs<__fp16> &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs<__fp16> &args) { return new GemmNative(args); } + [](const GemmArgs &args) { return (args._Ksize >= 8 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative(args); } }, { GemmMethod::GEMM_INTERLEAVED, "interleaved_fp16_mla_3VLx8", - [](const GemmArgs<__fp16> &args) { return (args._Ksize > 4); }, + [](const GemmArgs &args) { return (args._Ksize > 4); }, nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif @@ -72,21 +72,21 @@ static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = { GemmMethod::GEMM_INTERLEAVED, "hgemm_24x8", #ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - [](const GemmArgs<__fp16> &args) { return args._ci->has_fp16(); }, + [](const GemmArgs &args) { return args._ci->has_fp16(); }, #else nullptr, #endif nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, -#endif +#endif // aarch64 && FP16 #ifdef __aarch64__ { GemmMethod::GEMM_INTERLEAVED, "sgemm_12x8", nullptr, nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #elif defined(__arm__) { @@ -94,7 +94,7 @@ static const GemmImplementation<__fp16, __fp16> gemm_fp16_methods[] = { "sgemm_8x6", nullptr, nullptr, - [](const GemmArgs<__fp16> &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #else // not AArch64 or AArch32 # error Unknown Architecture @@ -114,9 +114,9 @@ const GemmImplementation<__fp16, __fp16> *gemm_implementation_list<__fp16, __fp1 } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &); -template KernelDescription get_gemm_method<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &); -template std::vector get_compatible_kernels<__fp16, __fp16, Nothing>(const GemmArgs<__fp16> &args, const Nothing &); +template UniqueGemmCommon<__fp16, __fp16> gemm<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &); +template std::vector get_compatible_kernels<__fp16, __fp16, Nothing>(const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp index dedcdb7655..cf91ee0652 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp @@ -33,17 +33,16 @@ #include "kernels/a32_sgemm_8x6.hpp" #include "kernels/a64_hybrid_fp32_mla_16x4.hpp" +#include "kernels/a64_native_fp32_mla_16x4.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp" #include "kernels/a64_sgemm_12x8.hpp" -#include "kernels/a64_sgemm_native_16x4.hpp" -#include "kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp" #include "kernels/a64_sgemv_pretransposed.hpp" #include "kernels/a64_sgemv_trans.hpp" #include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp" #include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp" #include "kernels/sve_native_fp32_mla_4VLx4.hpp" -#include "kernels/sve_smallK_fp32_mla_1VLx4.hpp" -#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp" namespace arm_gemm { @@ -52,88 +51,81 @@ static const GemmImplementation gemm_fp32_methods[] = { GemmMethod::GEMV_BATCHED, "gemv_batched", - [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); }, + [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); }, nullptr, - [](const GemmArgs &args) { return new GemvBatched(args); } + [](const GemmArgs &args) { return new GemvBatched(args); } }, #ifdef __aarch64__ { GemmMethod::GEMV_PRETRANSPOSED, "sgemv_pretransposed", - [](const GemmArgs &args) { return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1); }, + [](const GemmArgs &args) { return (args._Msize==1 && args._pretransposed_hint && args._nbatches==1); }, nullptr, - [](const GemmArgs &args) { return new GemvPretransposed(args); } + [](const GemmArgs &args) { return new GemvPretransposed(args); } }, { GemmMethod::GEMV_NATIVE_TRANSPOSED, "sgemv_trans", - [](const GemmArgs &args) { return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1); }, + [](const GemmArgs &args) { return (args._Msize==1 && !args._trA && !args._trB && args._nbatches==1); }, nullptr, - [](const GemmArgs &args) { return new GemvNativeTransposed(args); } + [](const GemmArgs &args) { return new GemvNativeTransposed(args); } }, #ifdef __ARM_FEATURE_SVE -// SVE smallk / native / hybrid methods -{ - GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_1VLx4", - [](const GemmArgs &args) { return (args._Ksize <= 24) && !args._trA && args._alpha==1.0f && args._pretransposed_hint; }, - nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } -}, +// SVE native / hybrid methods { GemmMethod::GEMM_HYBRID, "hybrid_fp32_mla_4VLx4", - [](const GemmArgs &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } -}, -{ - GemmMethod::GEMM_NATIVE, - "smallK_fp32_mla_1VLx4", - [](const GemmArgs &args) { return (args._Ksize <= 24) && !args._trA && !args._trB && args._alpha==1.0f; }, - nullptr, - [](const GemmArgs &args) { return new GemmNative(args); } + [](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_NATIVE, "native_fp32_mla_4VLx4", - [](const GemmArgs &args) { return (args._Ksize>4 && args._alpha==1.0f && !args._trA && !args._trB); }, - [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmNative(args); } + [](const GemmArgs &args) { return (args._Ksize>4 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative(args); } }, #endif // __ARM_FEATURE_SVE // NEON native / hybrid methods { GemmMethod::GEMM_HYBRID, - "sgemm_nativeA_pretransposeB_16x4", - [](const GemmArgs &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + "smallK_hybrid_fp32_mla_4x8", + [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0 && !args._trA && args._pretransposed_hint; }, + nullptr, + [](const GemmArgs &args) { return new GemmHybrid(args); } +}, +{ + GemmMethod::GEMM_HYBRID, + "smallK_hybrid_fp32_mla_4x6", + [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._trA && args._pretransposed_hint; }, + nullptr, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_fp32_mla_16x4", - [](const GemmArgs &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_NATIVE, - "sgemm_native_16x4", - [](const GemmArgs &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB); }, - [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmNative(args); } + "native_fp32_mla_16x4", + [](const GemmArgs &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative(args); } }, #ifdef __ARM_FEATURE_SVE { GemmMethod::GEMM_INTERLEAVED, "interleaved_fp32_mla_3VLx8", - [](const GemmArgs &args) { return (args._Ksize>4); }, + [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif // __ARM_FEATURE_SVE { @@ -141,7 +133,7 @@ static const GemmImplementation gemm_fp32_methods[] = "sgemm_12x8", nullptr, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif // __aarch64__ @@ -151,7 +143,7 @@ static const GemmImplementation gemm_fp32_methods[] = "sgemm_8x6", nullptr, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif // __arm__ { @@ -170,8 +162,8 @@ const GemmImplementation *gemm_implementation_list() } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon gemm(const GemmArgs &args, const Nothing &); -template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); -template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); +template UniqueGemmCommon gemm(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); +template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp index 436f55dee2..c3abb04db7 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2017-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include #include "arm_gemm.hpp" +#include "bias_adder.hpp" #include "ndrange.hpp" #include "utils.hpp" @@ -58,7 +59,7 @@ class GemmHybrid : public GemmCommon { const bool _trB; - const Tr _beta; + const Activation _act; /* Blocking info */ const unsigned int _k_block; @@ -70,7 +71,12 @@ class GemmHybrid : public GemmCommon { const NDRange<4> _window_range; - static unsigned int compute_k_block(const GemmArgs &args) { + static unsigned int compute_k_block(const GemmArgs &args) { + // Some kernels don't support append mode - these can't do K blocking at all. + if (!strategy::supports_append()) { + return args._Ksize; + } + if (args._cfg && args._cfg->inner_block_size) { return args._cfg->inner_block_size; } @@ -97,7 +103,7 @@ class GemmHybrid : public GemmCommon { return k_block; } - static unsigned int compute_n_block(const GemmArgs &args) { + static unsigned int compute_n_block(const GemmArgs &args) { if (args._cfg && args._cfg->outer_block_size) { return args._cfg->outer_block_size; } @@ -127,9 +133,10 @@ public: GemmHybrid & operator= (GemmHybrid &) = delete; /* Constructor */ - GemmHybrid(const GemmArgs &args) + GemmHybrid(const GemmArgs &args) : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), - _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta), + _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), + _act(args._act), _k_block(compute_k_block(args)), _n_block(compute_n_block(args)), _Mround(roundup(args._Msize, strategy::out_height())), _window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti) { } @@ -164,6 +171,9 @@ public: unsigned int kmax = std::min(k0 + _k_block, _Ksize); unsigned int kern_k = roundup(kmax-k0, strategy::k_unroll()); + const bool first_pass = (k0 == 0); + const bool last_pass = (kmax == _Ksize); + auto p = _window_range.iterator(start, end); if (p.done()) { @@ -190,8 +200,17 @@ public: strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda, b_panel, this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc, - (k0 == 0) ? _beta : static_cast(1), - (m_end - m_start), (nmax - n0), kmax-k0); + (m_end - m_start), (nmax - n0), kmax-k0, + (strategy::supports_bias() && first_pass && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr, + last_pass ? _act : Activation(), !first_pass); + + // Add bias externally if needed + if (!strategy::supports_bias() && this->_bias && first_pass) { + bias_adder(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc, + this->_bias + (multi * this->_bias_multi_stride) + n0, + (m_end - m_start), (nmax - n0)); + } + } while (p.next_dim1()); } } diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp index b4edece8d5..574ecef5b2 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp @@ -58,8 +58,6 @@ class GemmHybridQuantized : public GemmCommon { const bool _trB; - const Tr _beta; - /* Blocking info */ const unsigned int _k_block; const unsigned int _n_block; @@ -82,7 +80,7 @@ class GemmHybridQuantized : public GemmCommon { return _Nsize * _nmulti * sizeof(int32_t); } - static unsigned int compute_k_block(const GemmArgs &args) { + static unsigned int compute_k_block(const GemmArgs &args) { // We don't support K blocks as we only temporarily store 32 bit results. return args._Ksize; @@ -112,7 +110,7 @@ class GemmHybridQuantized : public GemmCommon { return k_block; } - static unsigned int compute_n_block(const GemmArgs &args) { + static unsigned int compute_n_block(const GemmArgs &args) { if (args._cfg && args._cfg->outer_block_size) { return args._cfg->outer_block_size; } @@ -142,9 +140,9 @@ public: GemmHybridQuantized & operator= (GemmHybridQuantized &) = delete; /* Constructor */ - GemmHybridQuantized(const GemmArgs &args, const ARequantizeLayer32 &qp) + GemmHybridQuantized(const GemmArgs &args, const ARequantizeLayer32 &qp) : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), - _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _beta(args._beta), + _nbatches(args._nbatches), _nmulti(args._nmulti), _trB(args._trB), _k_block(compute_k_block(args)), _n_block(compute_n_block(args)), _Mround(roundup(args._Msize, strategy::out_height())), _window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti), @@ -210,8 +208,8 @@ public: strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda, b_panel, result_buffer, (nmax-n0), - (k0 == 0) ? _beta : static_cast(1), - (m_end - m_start), (nmax - n0), kern_k); + (m_end - m_start), (nmax - n0), kern_k, + nullptr, Activation(), false); } { @@ -262,7 +260,7 @@ public: col_bias = reinterpret_cast(in_buffer); for (unsigned int i=0; i<_nmulti; i++) { - compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, 0); + compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, i, 0); } uintptr_t buffer_int = reinterpret_cast(in_buffer); @@ -295,8 +293,9 @@ public: col_bias = reinterpret_cast(in_buffer); } - void set_quantized_bias(const int32_t *bias) override { + void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override { _qp.bias = bias; + _qp.bias_multi_stride = bias_multi_stride; } }; diff --git a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp index 55d72f88cb..569d1f44ca 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp @@ -34,13 +34,13 @@ namespace arm_gemm { */ template struct GemmImplementation { - const GemmMethod method; - const char * name; - std::function &, const OutputStage &)> is_supported; - std::function &, const OutputStage &)> is_recommended; - std::function *(const GemmArgs &, const OutputStage &)> instantiate; + const GemmMethod method; + const char * name; + std::function is_supported; + std::function is_recommended; + std::function *(const GemmArgs &, const OutputStage &)> instantiate; - bool do_is_supported(const GemmArgs &args, const OutputStage &os) const { + bool do_is_supported(const GemmArgs &args, const OutputStage &os) const { if (is_supported != nullptr) { return is_supported(args, os); } else { @@ -48,7 +48,7 @@ struct GemmImplementation { } } - bool do_is_recommended(const GemmArgs &args, const OutputStage &os) const { + bool do_is_recommended(const GemmArgs &args, const OutputStage &os) const { if (is_recommended != nullptr) { return is_recommended(args, os); } else { @@ -56,7 +56,7 @@ struct GemmImplementation { } } - GemmCommon *do_instantiate(const GemmArgs &args, const OutputStage &os) const { + GemmCommon *do_instantiate(const GemmArgs &args, const OutputStage &os) const { return instantiate(args, os); } }; @@ -66,13 +66,13 @@ struct GemmImplementation { * unnecessary second argument. */ template struct GemmImplementation { - const GemmMethod method; - const char * name; - std::function &)> is_supported; - std::function &)> is_recommended; - std::function *(const GemmArgs &)> instantiate; + const GemmMethod method; + const char * name; + std::function is_supported; + std::function is_recommended; + std::function *(const GemmArgs &)> instantiate; - bool do_is_supported(const GemmArgs &args, const Nothing &) const { + bool do_is_supported(const GemmArgs &args, const Nothing &) const { if (is_supported != nullptr) { return is_supported(args); } else { @@ -80,7 +80,7 @@ struct GemmImplementation { } } - bool do_is_recommended(const GemmArgs &args, const Nothing &) const { + bool do_is_recommended(const GemmArgs &args, const Nothing &) const { if (is_recommended != nullptr) { return is_recommended(args); } else { @@ -88,7 +88,7 @@ struct GemmImplementation { } } - GemmCommon *do_instantiate(const GemmArgs &args, const Nothing &) const { + GemmCommon *do_instantiate(const GemmArgs &args, const Nothing &) const { return instantiate(args); } }; @@ -116,7 +116,7 @@ const GemmImplementation *gemm_implementation_list(); * reference. */ template -bool find_implementation(const GemmArgs &args, const OutputStage &os, const GemmImplementation * &impl) { +bool find_implementation(const GemmArgs &args, const OutputStage &os, const GemmImplementation * &impl) { auto gemms = gemm_implementation_list(); const GemmConfig *cfg = args._cfg; @@ -168,7 +168,7 @@ bool find_implementation(const GemmArgs &args, const OutputStage &os, cons } template -std::vector get_compatible_kernels(const GemmArgs &args, const OutputStage &os) { +std::vector get_compatible_kernels(const GemmArgs &args, const OutputStage &os) { std::vector res; /* Find out what the default implementation in so we can set the flag accordingly later. */ @@ -190,7 +190,7 @@ std::vector get_compatible_kernels(const GemmArgs &args } template -UniqueGemmCommon gemm(const GemmArgs &args, const OutputStage &os) { +UniqueGemmCommon gemm(const GemmArgs &args, const OutputStage &os) { const GemmImplementation *impl; if (find_implementation(args, os, impl)) { @@ -201,7 +201,7 @@ UniqueGemmCommon gemm(const GemmArgs &args, const OutputStage & } template -KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage &os) { +KernelDescription get_gemm_method(const GemmArgs &args, const OutputStage &os) { const GemmImplementation *impl; if (find_implementation(args, os, impl)) { diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp index 8062c3092a..e3b4416f68 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp @@ -38,7 +38,7 @@ static const GemmImplementation gemm_s16_methods[] = { "gemm_s16_12x8", nullptr, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, { GemmMethod::DEFAULT, @@ -55,9 +55,9 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); -template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); -template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); +template UniqueGemmCommon gemm(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); +template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp index 49241a7129..a3446b9ddc 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp @@ -48,66 +48,66 @@ static const GemmImplementation gemm_s8_methods[] = { { GemmMethod::GEMM_HYBRID, "smallK_hybrid_s8s32_dot_1VLx8", - [](const GemmArgs &args) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_s8s32_dot_4VLx4", - [](const GemmArgs &args) { return args._Ksize>=16 && args._alpha==1 && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return args._Ksize>=16 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_NATIVE, "native_s8s32_dot_4VLx4", - [](const GemmArgs &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); }, - [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, - [](const GemmArgs &args) { return new GemmNative(args); } + [](const GemmArgs &args) { return (args._Ksize>=16 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, + [](const GemmArgs &args) { return new GemmNative(args); } }, { GemmMethod::GEMM_INTERLEAVED, "interleaved_s8s32_dot_3VLx8", - [](const GemmArgs &args) { return (args._Ksize>4) && args._alpha==1 && (args._beta == 0 || args._beta==1); }, + [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif { GemmMethod::GEMM_HYBRID, "smallK_hybrid_s8s32_dot_4x8", - [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "smallK_hybrid_s8s32_dot_4x6", - [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_s8s32_dot_16x4", - [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_s8_12x8", - [](const GemmArgs &args) { return args._ci->has_dotprod() && args._alpha==1 && (args._beta==0 || args._beta==1); }, + [](const GemmArgs &args) { return args._ci->has_dotprod(); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_s8_4x4", nullptr, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, { GemmMethod::DEFAULT, @@ -124,9 +124,9 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); -template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); -template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); +template UniqueGemmCommon gemm(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); +template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp index faff9acd2e..efd984561d 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp @@ -68,8 +68,7 @@ class GemmInterleaved : public GemmCommon { const bool _trA; const bool _trB; - const Tr _alpha; - const Tr _beta; + const Activation _act; const int _maxthreads; int _nthreads; @@ -297,9 +296,14 @@ class GemmInterleaved : public GemmCommon { #ifdef CYCLE_PROFILING auto p=prof.ScopedProfiler(PROFILE_MERGE, (strategy::out_height() * bblocks * strategy::out_width() * sizeof(Tr))); #endif + /* Only activate on last pass, only add bias on first pass, ask for accumulation on any non-first pass */ + const bool first_pass = current.k0()==0; + const bool last_pass = current.kmax()==_Ksize; + strat.transforms.Merge(this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride), c_panel, this->_ldc, y, ymax, current.x0(), current.xmax(), - _alpha, (current.k0()==0 ? _beta : static_cast(1))); + ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr), + (last_pass ? _act : Activation()), !first_pass); } } } @@ -317,10 +321,10 @@ public: GemmInterleaved & operator= (GemmInterleaved &) = delete; /* Constructor */ - GemmInterleaved(const GemmArgs &args) + GemmInterleaved(const GemmArgs &args) : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), _nbatches(args._nbatches), _nmulti(args._nmulti), _trA(args._trA), _trB(args._trB), - _alpha(args._alpha), _beta(args._beta), _maxthreads(args._maxthreads), _nthreads(args._maxthreads), + _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads), _pretransposed(args._pretransposed_hint) { const unsigned int L1_size = _ci->get_L1_cache_size(); const unsigned int L2_size = _ci->get_L2_cache_size(); diff --git a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp index ba9163b29b..fe6ebef045 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_native.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_native.hpp @@ -54,7 +54,7 @@ class GemmNative : public GemmCommon { const unsigned int _nbatches; const unsigned int _nmultis; - const Tr _beta; + const Activation _act; const CPUInfo * const _ci; @@ -63,11 +63,11 @@ class GemmNative : public GemmCommon { const NDRange<4> _window_range; - static unsigned int compute_k_block(const GemmArgs &args) { + static unsigned int compute_k_block(const GemmArgs &args) { return args._Ksize; } - static unsigned int compute_n_block(const GemmArgs &args) { + static unsigned int compute_n_block(const GemmArgs &args) { if ((args._cfg != nullptr) && args._cfg->outer_block_size > 0) { return args._cfg->outer_block_size; } else { @@ -79,10 +79,10 @@ public: GemmNative(GemmNative &) = delete; GemmNative & operator= (GemmNative &) = delete; - GemmNative(const GemmArgs &args) + GemmNative(const GemmArgs &args) : _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize), _nbatches(args._nbatches), _nmultis(args._nmulti), - _beta(args._beta), _ci(args._ci), + _act(args._act), _ci(args._ci), _k_block(compute_k_block(args)), _n_block(compute_n_block(args)), _window_range(iceildiv(_Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmultis) { } @@ -127,7 +127,16 @@ public: strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (y0 * this->_lda), this->_lda, this->_Bptr + (multi * this->_B_multi_stride) + n0, this->_ldb, this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (y0 * this->_ldc) + n0, this->_ldc, - _beta, (ymax-y0), (nmax - n0), _Ksize); + (ymax-y0), (nmax-n0), _Ksize, + (strategy::supports_bias() && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr, + _act, false); + + // Add bias externally if needed + if (!strategy::supports_bias() && this->_bias) { + bias_adder(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (y0 * this->_ldc) + n0, this->_ldc, + this->_bias + (multi * this->_bias_multi_stride) + n0, + (ymax - y0), (nmax - n0)); + } } while (p.next_dim1()); } }; diff --git a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp index a402332552..079c04ae06 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp @@ -42,45 +42,45 @@ static const GemmImplementation gemm_quint { GemmMethod::GEMM_HYBRID_QUANTIZED, "smallK_hybrid_u8u32_dot_1VLx8", - [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } }, { GemmMethod::GEMM_HYBRID_QUANTIZED, "hybrid_u8u32_dot_4VLx4", - [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs &args, const ARequantizeLayer32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } }, #endif { GemmMethod::GEMM_HYBRID_QUANTIZED, "smallK_hybrid_u8u32_dot_4x8", - [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } }, { GemmMethod::GEMM_HYBRID_QUANTIZED, "smallK_hybrid_u8u32_dot_4x6", - [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } }, { GemmMethod::GEMM_HYBRID_QUANTIZED, "hybrid_u8u32_dot_16x4", - [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Nsize<=256 && args._Ksize>128; }, - [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args, const ARequantizeLayer32 &) { return args._Nsize<=256 && args._Ksize>128; }, + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new GemmHybridQuantized(args, qp); } }, { GemmMethod::QUANTIZE_WRAPPER, "quantized_wrapper", nullptr, nullptr, - [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new QuantizeWrapper(args, qp); } + [](const GemmArgs &args, const ARequantizeLayer32 &qp) { return new QuantizeWrapper(args, qp); } }, { GemmMethod::DEFAULT, @@ -96,9 +96,9 @@ const GemmImplementation *gemm_implementat return gemm_quint8_methods; } -template UniqueGemmCommon gemm(const GemmArgs &args, const ARequantizeLayer32 &os); -template KernelDescription get_gemm_method(const GemmArgs &args, const ARequantizeLayer32 &os); -template std::vector get_compatible_kernels(const GemmArgs &args, const ARequantizeLayer32 &os); +template UniqueGemmCommon gemm(const GemmArgs &args, const ARequantizeLayer32 &os); +template KernelDescription get_gemm_method(const GemmArgs &args, const ARequantizeLayer32 &os); +template std::vector get_compatible_kernels(const GemmArgs &args, const ARequantizeLayer32 &os); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp index 408911b6d5..85a8a6720a 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp @@ -38,7 +38,7 @@ static const GemmImplementation gemm_u16_methods[] = { "gemm_u16_12x8", nullptr, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, { GemmMethod::DEFAULT, @@ -55,9 +55,9 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); -template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); -template std::vector get_compatible_kernels(const GemmArgs &args, const Nothing &); +template UniqueGemmCommon gemm(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); +template std::vector get_compatible_kernels(const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp index f5a30720d2..aead814d7e 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp @@ -48,66 +48,66 @@ static const GemmImplementation gemm_u8_methods[] = { { GemmMethod::GEMM_HYBRID, "smallK_hybrid_u8u32_dot_1VLx8", - [](const GemmArgs &args) { return args._Ksize<=64 && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Ksize<=64 && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_u8u32_dot_4VLx4", - [](const GemmArgs &args) { return args._Ksize>=16 && args._alpha==1 && !args._trA && args._pretransposed_hint; }, - [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return args._Ksize>=16 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_NATIVE, "native_u8u32_dot_4VLx4", - [](const GemmArgs &args) { return (args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB); }, - [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, - [](const GemmArgs &args) { return new GemmNative(args); } + [](const GemmArgs &args) { return (args._Ksize>=16 && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); }, + [](const GemmArgs &args) { return new GemmNative(args); } }, { GemmMethod::GEMM_INTERLEAVED, "interleaved_u8u32_dot_3VLx8", - [](const GemmArgs &args) { return (args._Ksize>4) && args._alpha==1 && (args._beta==0 || args._beta==1); }, + [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif { GemmMethod::GEMM_HYBRID, "smallK_hybrid_u8u32_dot_4x8", - [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "smallK_hybrid_u8u32_dot_4x6", - [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && args._alpha==1 && args._beta==0 && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._trA && args._pretransposed_hint; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, "hybrid_u8u32_dot_16x4", - [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && args._alpha==1 && !args._trA && !args._trB && args._pretransposed_hint; }, - [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16 && !args._trA && !args._trB && args._pretransposed_hint; }, + [](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; }, + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_u8_12x8", - [](const GemmArgs &args) { return args._ci->has_dotprod() && args._alpha==1 && (args._beta==0 || args._beta==1); }, + [](const GemmArgs &args) { return args._ci->has_dotprod(); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, { GemmMethod::GEMM_INTERLEAVED, "gemm_u8_4x4", nullptr, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, { GemmMethod::DEFAULT, @@ -124,9 +124,9 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); -template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); -template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); +template UniqueGemmCommon gemm(const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); +template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp index 4453ee8243..be2f5614be 100644 --- a/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemv_batched.hpp @@ -36,9 +36,9 @@ private: UniqueGemmCommon _subgemm = nullptr; public: - GemvBatched(const GemmArgs &args) { + GemvBatched(const GemmArgs &args) { /* Just create a subgemm with batches->M */ - GemmArgs newargs = args; + GemmArgs newargs = args; newargs._Msize = args._nbatches; newargs._nbatches = 1; newargs._cfg = nullptr; @@ -47,13 +47,15 @@ public: void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, const int B_multi_stride, - Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override { - UNUSED(lda); - UNUSED(ldc); + Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const Tr *bias, const int bias_multi_stride) override { /* A and C's batch stride becomes their new row stride. New batch stride is 0 as nbatches for subgemm is always 1. */ _subgemm->set_arrays(A, A_batch_stride, 0, A_multi_stride, B, ldb, B_multi_stride, - C, C_batch_stride, 0, C_multi_stride); + C, C_batch_stride, 0, C_multi_stride, + bias, bias_multi_stride); + UNUSED(lda); + UNUSED(ldc); } unsigned int get_window_size() const override { diff --git a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp index 55b1f9bbe6..49681ec404 100644 --- a/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemv_native_transposed.hpp @@ -53,7 +53,7 @@ class GemvNativeTransposed : public GemmCommon { const unsigned int _nmultis; - const Tr _beta; + const Activation _act; const CPUInfo * const _ci; @@ -64,8 +64,8 @@ public: GemvNativeTransposed(GemvNativeTransposed &) = delete; GemvNativeTransposed & operator= (GemvNativeTransposed &) = delete; - GemvNativeTransposed(const GemmArgs &args) - : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _beta(args._beta), _ci(args._ci) { + GemvNativeTransposed(const GemmArgs &args) + : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _act(args._act), _ci(args._ci) { /* For now don't do any blocking. TODO: figure out if we should. */ m_block = _Ksize; n_block = _Nsize; @@ -111,7 +111,18 @@ public: strat.kernel(this->_Bptr + (multi * this->_B_multi_stride) + (m0 * this->_ldb) + n0, this->_Aptr + (multi * this->_A_multi_stride) + m0, this->_Cptr + (multi * this->_C_multi_stride) + n0, - _beta, this->_ldb, (mmax-m0), (nmax-n0)); + static_cast(0), this->_ldb, (mmax-m0), (nmax-n0)); + + // Handle activation separately for now + if (this->_bias) { + activator(this->_Cptr + (multi * this->_C_multi_stride) + n0, 0, + this->_bias + (multi * this->_bias_multi_stride) + n0, + _act, 1, (nmax-n0)); + } else { + activator(this->_Cptr + (multi * this->_C_multi_stride) + n0, 0, + static_cast(nullptr), + _act, 1, (nmax-n0)); + } } } } diff --git a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp index 92064180a2..26fdfba8ff 100644 --- a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp @@ -26,7 +26,7 @@ #include #include "arm_gemm.hpp" - +#include "bias_adder.hpp" #include "mergeresults.hpp" #include "transform.hpp" @@ -53,7 +53,7 @@ class GemvPretransposed : public GemmCommon { const bool _trB; - const Tr _beta; + const Activation _act; const CPUInfo * const _ci; @@ -68,8 +68,8 @@ public: GemvPretransposed(GemvPretransposed &) = delete; GemvPretransposed & operator= (GemvPretransposed &) = delete; - GemvPretransposed(const GemmArgs &args) - : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _beta(args._beta), _ci(args._ci), + GemvPretransposed(const GemmArgs &args) + : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _trB(args._trB), _act(args._act), _ci(args._ci), _buffer_per_multi(_Ksize * iceildiv(_Nsize, strategy::A_interleave()) * strategy::A_interleave()) { /* For now don't do any blocking. TODO: figure out if we should. */ if (args._cfg && args._cfg->inner_block_size) { @@ -128,7 +128,18 @@ public: (_Ksize * strategy::A_interleave()), this->_Aptr + (multi * this->_A_multi_stride) + m0, this->_Cptr + (multi * this->_C_multi_stride) + n, - _beta, (mmax-m0), (nmax-n)); + static_cast(0), (mmax-m0), (nmax-n)); + + // Handle activation separately for now + if (this->_bias) { + activator(this->_Cptr + (multi * this->_C_multi_stride) + n, 0, + this->_bias + (multi * this->_bias_multi_stride) + n, + _act, 1, (nmax-n)); + } else { + activator(this->_Cptr + (multi * this->_C_multi_stride) + n, 0, + static_cast(nullptr), + _act, 1, (nmax-n)); + } } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp index cc205dc6e3..941281f24d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp @@ -61,7 +61,9 @@ public: kern_type kernel = a64_gemm_s8_12x8; gemm_s8_12x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto mod = ci->get_cpu_model(); + + if (mod == CPUModel::A55r1) { kernel = a64_gemm_s8_12x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp index 9032ba67b3..c228b8b858 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp @@ -25,8 +25,6 @@ #ifdef __aarch64__ -#include "arm_gemm.hpp" - #include "../std_transforms_fixed.hpp" namespace arm_gemm { @@ -71,7 +69,9 @@ public: kern_type kernel = a64_gemm_u8_12x8; gemm_u8_12x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto mod = ci->get_cpu_model(); + + if (mod == CPUModel::A55r1) { kernel = a64_gemm_u8_12x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp index 5b850b7a20..283fea1102 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp @@ -25,8 +25,6 @@ #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) -#include "arm_gemm.hpp" - #include "../std_transforms_fixed.hpp" namespace arm_gemm { @@ -66,7 +64,9 @@ public: kern_type kernel = a64_hgemm_asimd_24x8; hgemm_24x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto model = ci->get_cpu_model(); + + if (model == CPUModel::A55r1) { kernel = a64_hgemm_asimd_24x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp index 560593958a..1ce934d413 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, float, int, int, int); -void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, float, int, int, int); +void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); +void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); class hybrid_fp32_mla_16x4 { @@ -41,10 +41,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsFixed transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp index 7261761d7e..e3325f7728 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp @@ -25,20 +25,41 @@ #include +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -458,56 +484,47 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "temploadreg3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -931,9 +948,27 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v23.4s, v11.4s, v1.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" "str q20, [c_ptr1]\n" @@ -947,7 +982,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -965,131 +1000,118 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v24.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v25.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "movi v26.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "movi v27.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ins v14.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" "fmla v16.4s, v8.4s, v0.s[0]\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" "fmla v20.4s, v8.4s, v1.s[0]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "fmla v24.4s, v8.4s, v2.s[0]\n" - "ldr d4, [%[a_ptr0]]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "fmla v17.4s, v9.4s, v0.s[0]\n" - "ldr temploadreg0, [%[a_ptr0], #0x8]\n" + "ldr d4, [%[a_ptr0]]\n" "fmla v21.4s, v9.4s, v1.s[0]\n" - "ldr d5, [a_ptr1]\n" + "ldr temploadreg0, [%[a_ptr0], #0x8]\n" "fmla v25.4s, v9.4s, v2.s[0]\n" - "ldr temploadreg1, [a_ptr1, #0x8]\n" + "ldr d5, [a_ptr1]\n" "fmla v18.4s, v10.4s, v0.s[0]\n" - "ldr d6, [a_ptr2]\n" + "ldr temploadreg1, [a_ptr1, #0x8]\n" "fmla v22.4s, v10.4s, v1.s[0]\n" - "ldr temploadreg2, [a_ptr2, #0x8]\n" + "ldr d6, [a_ptr2]\n" "fmla v26.4s, v10.4s, v2.s[0]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg2, [a_ptr2, #0x8]\n" "fmla v19.4s, v11.4s, v0.s[0]\n" - "ins v4.d[1], temploadreg0\n" + "ldr d8, [%[b_ptr0]]\n" "fmla v23.4s, v11.4s, v1.s[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ins v4.d[1], temploadreg0\n" "fmla v27.4s, v11.4s, v2.s[0]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "fmla v16.4s, v12.4s, v0.s[1]\n" - "ins v5.d[1], temploadreg1\n" + "ldr d9, [%[b_ptr0], #0x10]\n" "fmla v20.4s, v12.4s, v1.s[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ins v5.d[1], temploadreg1\n" "fmla v24.4s, v12.4s, v2.s[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "fmla v17.4s, v13.4s, v0.s[1]\n" - "ins v6.d[1], temploadreg2\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "fmla v21.4s, v13.4s, v1.s[1]\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "ins v6.d[1], temploadreg2\n" "fmla v25.4s, v13.4s, v2.s[1]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "fmla v18.4s, v14.4s, v0.s[1]\n" - "ins v15.d[1], temploadreg3\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "fmla v22.4s, v14.4s, v1.s[1]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ins v15.d[1], temploadreg3\n" "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" "fmla v19.4s, v15.4s, v0.s[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x48]\n" + "ins v8.d[1], temploadreg0\n" "fmla v23.4s, v15.4s, v1.s[1]\n" - "ldr d13, [%[b_ptr0], #0x50]\n" + "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" "fmla v16.4s, v8.4s, v0.s[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "fmla v20.4s, v8.4s, v1.s[2]\n" @@ -1113,41 +1135,39 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v23.4s, v11.4s, v1.s[2]\n" "ins v15.d[1], temploadreg3\n" "fmla v27.4s, v11.4s, v2.s[2]\n" - "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - "fmla v16.4s, v12.4s, v0.s[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" "ins v8.d[1], temploadreg0\n" - "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" + "ins v9.d[1], temploadreg1\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla v16.4s, v8.4s, v4.s[0]\n" - "ins v9.d[1], temploadreg1\n" - "fmla v20.4s, v8.4s, v5.s[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" "ins v10.d[1], temploadreg2\n" - "add a_ptr1, a_ptr1, #0x20\n" "fmla v17.4s, v9.4s, v4.s[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "fmla v21.4s, v9.4s, v5.s[0]\n" @@ -1163,70 +1183,71 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v19.4s, v11.4s, v4.s[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "fmla v23.4s, v11.4s, v5.s[0]\n" - "ldr d1, [a_ptr1, #-0x10]\n" - "fmla v27.4s, v11.4s, v6.s[0]\n" "ins v13.d[1], temploadreg1\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ins v14.d[1], temploadreg2\n" "fmla v16.4s, v12.4s, v4.s[1]\n" - "ldr temploadreg1, [a_ptr1, #-0x8]\n" + "ldr d8, [%[b_ptr0]]\n" "fmla v20.4s, v12.4s, v5.s[1]\n" - "ins v14.d[1], temploadreg2\n" + "ins v0.d[1], temploadreg0\n" "fmla v24.4s, v12.4s, v6.s[1]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "fmla v17.4s, v13.4s, v4.s[1]\n" - "ins v0.d[1], temploadreg0\n" + "ldr d9, [%[b_ptr0], #0x10]\n" "fmla v21.4s, v13.4s, v5.s[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "fmla v25.4s, v13.4s, v6.s[1]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "fmla v18.4s, v14.4s, v4.s[1]\n" - "ins v1.d[1], temploadreg1\n" + "ins v15.d[1], temploadreg3\n" "fmla v22.4s, v14.4s, v5.s[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "fmla v26.4s, v14.4s, v6.s[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x20\n" - "ins v15.d[1], temploadreg3\n" - "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - "ldr d2, [a_ptr2, #-0x10]\n" - "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "ldr d12, [%[b_ptr0], #0x40]\n" + "ins v8.d[1], temploadreg0\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla v19.4s, v15.4s, v4.s[1]\n" - "ldr temploadreg2, [a_ptr2, #-0x8]\n" + "ldr d1, [a_ptr1, #-0x10]\n" "fmla v23.4s, v15.4s, v5.s[1]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ldr temploadreg1, [a_ptr1, #-0x8]\n" "fmla v27.4s, v15.4s, v6.s[1]\n" - "ldr d12, [%[b_ptr0], #0x40]\n" - "ins v8.d[1], temploadreg0\n" - "ins v2.d[1], temploadreg2\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "fmla v16.4s, v8.4s, v4.s[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "fmla v20.4s, v8.4s, v5.s[2]\n" - "ins v9.d[1], temploadreg1\n" + "ins v1.d[1], temploadreg1\n" "fmla v24.4s, v8.4s, v6.s[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x58]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "ins v10.d[1], temploadreg2\n" - "fmla v17.4s, v9.4s, v4.s[2]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "fmla v21.4s, v9.4s, v5.s[2]\n" + "add a_ptr2, a_ptr2, #0x20\n" "ldr d15, [%[b_ptr0], #0x70]\n" - "fmla v25.4s, v9.4s, v6.s[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "ldr d2, [a_ptr2, #-0x10]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "ldr temploadreg2, [a_ptr2, #-0x8]\n" + "ins v9.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ins v11.d[1], temploadreg3\n" - "fmla v18.4s, v10.4s, v4.s[2]\n" + "ins v2.d[1], temploadreg2\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" "ins v12.d[1], temploadreg0\n" - "fmla v26.4s, v10.4s, v6.s[2]\n" - "ins v13.d[1], temploadreg1\n" "fmla v19.4s, v11.4s, v4.s[2]\n" - "ins v14.d[1], temploadreg2\n" + "ins v10.d[1], temploadreg2\n" "fmla v23.4s, v11.4s, v5.s[2]\n" - "ins v15.d[1], temploadreg3\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "fmla v27.4s, v11.4s, v6.s[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x100\n" + "ins v13.d[1], temploadreg1\n" "fmla v16.4s, v12.4s, v4.s[3]\n" + "ins v15.d[1], temploadreg3\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ins v14.d[1], temploadreg2\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x100\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "fmla v20.4s, v12.4s, v5.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" @@ -1258,13 +1279,13 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "ins v14.d[1], temploadreg2\n" "b.ne 3b\n" "2:\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [c_ptr1]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" @@ -1554,13 +1575,39 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v27.4s, v11.4s, v2.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" "str q22, [c_ptr1, #0x20]\n" "str q23, [c_ptr1, #0x30]\n" "str q24, [c_ptr2]\n" @@ -1576,7 +1623,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1599,89 +1646,72 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q3, [a_ptr3]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v24.4s, #0\n" + "mov v28.16b, v16.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v25.4s, #0\n" + "mov v29.16b, v17.16b\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v26.4s, #0\n" + "mov v30.16b, v18.16b\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "movi v27.4s, #0\n" + "mov v31.16b, v19.16b\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "movi v28.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v29.4s, #0\n" - "ins v14.d[1], temploadreg2\n" - "movi v30.4s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "movi v31.4s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" + "ins v14.d[1], temploadreg2\n" "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "fmul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "fmul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "fmul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "fmul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -2303,16 +2333,50 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v31.4s, v11.4s, v3.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" "str q25, [c_ptr2, #0x10]\n" "str q26, [c_ptr2, #0x20]\n" "str q27, [c_ptr2, #0x30]\n" @@ -2331,7 +2395,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp index 504769b9f0..d60d08a672 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp @@ -25,20 +25,41 @@ #include +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -307,55 +333,46 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "c_ptr1 .req X1\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -635,9 +652,27 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v23.4s, v11.4s, v1.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" "str q20, [c_ptr1]\n" @@ -647,7 +682,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -661,71 +696,58 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v24.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v25.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" - "movi v26.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v27.4s, #0\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -1098,13 +1120,39 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v27.4s, v11.4s, v2.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" "str q22, [c_ptr1, #0x20]\n" "str q23, [c_ptr1, #0x30]\n" "str q24, [c_ptr2]\n" @@ -1116,7 +1164,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1135,87 +1183,70 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q3, [a_ptr3]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v24.4s, #0\n" + "mov v28.16b, v16.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v25.4s, #0\n" + "mov v29.16b, v17.16b\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v26.4s, #0\n" + "mov v30.16b, v18.16b\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "movi v27.4s, #0\n" + "mov v31.16b, v19.16b\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v28.4s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "movi v29.4s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "movi v30.4s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "fmul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "fmul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "fmul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "fmul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -1681,16 +1712,50 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v31.4s, v11.4s, v3.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" "str q25, [c_ptr2, #0x10]\n" "str q26, [c_ptr2, #0x20]\n" "str q27, [c_ptr2, #0x30]\n" @@ -1705,7 +1770,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp index c8934dff8a..5a6fabcfa9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class hybrid_s8s32_dot_16x4 { @@ -41,10 +41,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp index 3f72e4bea1..3ecf0151aa 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in for (int x0=0; x0(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -476,178 +454,144 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X3\n" "temploadreg2 .req X4\n" "temploadreg3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" + "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "movi v20.4s, #0\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "movi v21.4s, #0\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" + "movi v22.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" + "movi v23.4s, #0\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ins v14.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "subs %[loops], %[loops], #0x1\n" "ldr d11, [%[b_ptr0], #0x30]\n" - "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" "ins v15.d[1], temploadreg3\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "add a_ptr1, a_ptr1, #0x20\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v1.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" @@ -656,55 +600,55 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -714,223 +658,223 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "b.ne 3b\n" - "2:\n" "ins v14.d[1], temploadreg2\n" - "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr d13, [%[b_ptr0], #0x50]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -939,43 +883,43 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -992,7 +936,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -1006,228 +950,183 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X5\n" "temploadreg2 .req X6\n" "temploadreg3 .req X7\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v25.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v26.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ldr q1, [a_ptr1]\n" "movi v27.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" "ins v14.d[1], temploadreg2\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x20\n" @@ -1235,65 +1134,65 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d2, [a_ptr2, #-0x10]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ins v2.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -1304,275 +1203,275 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1581,56 +1480,56 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1653,7 +1552,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1670,364 +1569,309 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X7\n" "temploadreg2 .req X8\n" "temploadreg3 .req X9\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q3, [a_ptr3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v25.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v26.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr q1, [a_ptr1]\n" "movi v27.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ins v14.d[1], temploadreg2\n" "movi v28.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v29.4s, #0\n" - "ins v14.d[1], temploadreg2\n" + "ldr q2, [a_ptr2]\n" "movi v30.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr3, a_ptr2, %[lda]\n" "movi v31.4s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add a_ptr3, a_ptr3, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" - "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" - "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" "ldr q3, [a_ptr3]\n" - "ldr q8, [%[b_ptr0]]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr3, a_ptr3, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr d2, [a_ptr2, #-0x10]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ins v2.d[1], temploadreg2\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr d3, [a_ptr3, #-0x10]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" "ldr temploadreg3, [a_ptr3, #-0x8]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ins v3.d[1], temploadreg3\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" "ins v14.d[1], temploadreg2\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" @@ -2035,319 +1879,319 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -2356,69 +2200,69 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -2447,7 +2291,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp index 1ad924612e..b48b674621 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ for (int x0=0; x0(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -325,316 +304,283 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" + "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "movi v20.4s, #0\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "movi v21.4s, #0\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" + "movi v22.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" + "movi v23.4s, #0\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -643,43 +589,43 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -692,7 +638,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -702,421 +648,378 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v25.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" + "ldr q1, [a_ptr1]\n" "movi v26.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v27.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1125,56 +1028,56 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1193,7 +1096,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1206,526 +1109,473 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q3, [a_ptr3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v25.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr q1, [a_ptr1]\n" "movi v26.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v27.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q2, [a_ptr2]\n" "movi v28.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr3, a_ptr2, %[lda]\n" "movi v29.4s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "ldr q3, [a_ptr3]\n" "movi v30.4s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "movi v31.4s, #0\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" - "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" - "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" - "ldr q3, [a_ptr3]\n" - "ldr q8, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr3, a_ptr3, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q3, [a_ptr3, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1734,69 +1584,69 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1821,7 +1671,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp index 48731efc57..5295650e7b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class hybrid_u8u32_dot_16x4 { @@ -41,7 +41,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp index e343d58fa0..e8ed0c311e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, for (int x0=0; x0(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -478,7 +475,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "temploadreg3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -503,31 +500,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -535,33 +523,33 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "subs %[loops], %[loops], #0x1\n" @@ -571,83 +559,83 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x20\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v1.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" @@ -656,55 +644,55 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -723,211 +711,211 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr d13, [%[b_ptr0], #0x50]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -939,16 +927,16 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -967,14 +955,14 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -992,7 +980,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -1010,7 +998,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1042,41 +1030,28 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -1085,149 +1060,149 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x20\n" @@ -1235,65 +1210,65 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d2, [a_ptr2, #-0x10]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ins v2.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -1313,263 +1288,263 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1581,22 +1556,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1618,18 +1593,18 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1653,7 +1628,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1676,7 +1651,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1714,51 +1689,34 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -1767,261 +1725,261 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr d2, [a_ptr2, #-0x10]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ins v2.d[1], temploadreg2\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr d3, [a_ptr3, #-0x10]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" "ldr temploadreg3, [a_ptr3, #-0x8]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ins v3.d[1], temploadreg3\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" @@ -2036,315 +1994,315 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -2356,28 +2314,28 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -2402,22 +2360,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -2447,7 +2405,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp index 6301fa5657..23d919a64c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint for (int x0=0; x0(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -327,7 +324,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "c_ptr1 .req X1\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -351,287 +348,278 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -643,16 +631,16 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -671,14 +659,14 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -692,7 +680,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -706,7 +694,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -736,190 +724,177 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" @@ -927,193 +902,193 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1125,22 +1100,22 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1162,18 +1137,18 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1193,7 +1168,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1212,7 +1187,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1248,236 +1223,219 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q3, [a_ptr3, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" @@ -1486,243 +1444,243 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1734,28 +1692,28 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1780,22 +1738,22 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1821,7 +1779,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp new file mode 100644 index 0000000000..a86e8ec068 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2018-2019 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +namespace arm_gemm +{ + +// Actual kernel implementations +void a64_native_fp32_mla_16x4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); + +class native_fp32_mla_16x4 +{ +public: + typedef float operand_type; + typedef float result_type; + + typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); + + /* Kernel blocking parameters */ + static constexpr unsigned int out_height() + { + return 4; + } + + static unsigned int out_width() + { + return 16; + } + + static constexpr unsigned int k_unroll() + { + return 1; + } + + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + + + + // Default to the generic kernel + kern_type kernel=a64_native_fp32_mla_16x4; + + native_fp32_mla_16x4(const CPUInfo *ci) { UNUSED(ci); } +}; + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp new file mode 100644 index 0000000000..82e7333ee3 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp @@ -0,0 +1,1708 @@ +/* + * Copyright (c) 2018-2019 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include + +#include "arm_gemm.hpp" + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_native_fp32_mla_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { + const long loops_count = ((K + 4) / 8) - 1; + K -= loops_count * 8; + const long regs_count = (K / 4) - 1; + K -= (regs_count + 1) * 4; + const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y=0; y(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 2: + __asm __volatile ( + "a_ptr1 .req X0\n" + "c_ptr1 .req X1\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "ldr q1, [a_ptr1]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov v22.16b, v18.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v23.16b, v19.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "str q20, [c_ptr1]\n" + "str q21, [c_ptr1, #0x10]\n" + "str q22, [c_ptr1, #0x20]\n" + "str q23, [c_ptr1, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq c_ptr1\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" + ); + break; + case 3: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "c_ptr1 .req X2\n" + "c_ptr2 .req X3\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "mov v24.16b, v16.16b\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "mov v25.16b, v17.16b\n" + "ldr q1, [a_ptr1]\n" + "mov v22.16b, v18.16b\n" + "ldr q2, [a_ptr2]\n" + "mov v23.16b, v19.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v26.16b, v18.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "mov v27.16b, v19.16b\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add a_ptr2, a_ptr2, #0x20\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q2, [a_ptr2, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [c_ptr2]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr s2, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x4\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" + "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" + "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "str q22, [c_ptr1, #0x20]\n" + "str q23, [c_ptr1, #0x30]\n" + "str q24, [c_ptr2]\n" + "str q25, [c_ptr2, #0x10]\n" + "str q26, [c_ptr2, #0x20]\n" + "str q27, [c_ptr2, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" + ); + break; + default: + case 4: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "c_ptr1 .req X3\n" + "c_ptr2 .req X4\n" + "c_ptr3 .req X5\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "mov v24.16b, v16.16b\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "mov v25.16b, v17.16b\n" + "ldr q1, [a_ptr1]\n" + "mov v22.16b, v18.16b\n" + "ldr q2, [a_ptr2]\n" + "mov v23.16b, v19.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v26.16b, v18.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "mov v27.16b, v19.16b\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "mov v28.16b, v16.16b\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "mov v29.16b, v17.16b\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "mov v30.16b, v18.16b\n" + "ldr q3, [a_ptr3]\n" + "mov v31.16b, v19.16b\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add a_ptr3, a_ptr3, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q7, [a_ptr3]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "add a_ptr2, a_ptr2, #0x20\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x20\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q2, [a_ptr2, #-0x10]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q3, [a_ptr3, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v28.4s, v8.4s, v7.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "fmla v29.4s, v9.4s, v7.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "fmla v30.4s, v10.4s, v7.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "fmla v31.4s, v11.4s, v7.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "fmla v28.4s, v12.4s, v7.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "fmla v29.4s, v13.4s, v7.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "fmla v30.4s, v14.4s, v7.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "fmla v31.4s, v15.4s, v7.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v28.4s, v8.4s, v7.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v29.4s, v9.4s, v7.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v30.4s, v10.4s, v7.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v31.4s, v11.4s, v7.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v28.4s, v12.4s, v7.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v29.4s, v13.4s, v7.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v30.4s, v14.4s, v7.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "fmla v31.4s, v15.4s, v7.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [c_ptr2]\n" + "prfm PSTL1KEEP, [c_ptr3]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q7, [a_ptr3]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x10\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v28.4s, v8.4s, v7.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "fmla v29.4s, v9.4s, v7.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "fmla v30.4s, v10.4s, v7.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "fmla v31.4s, v11.4s, v7.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "fmla v28.4s, v12.4s, v7.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "fmla v29.4s, v13.4s, v7.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "fmla v30.4s, v14.4s, v7.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "fmla v31.4s, v15.4s, v7.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v28.4s, v8.4s, v7.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v29.4s, v9.4s, v7.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v30.4s, v10.4s, v7.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v31.4s, v11.4s, v7.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v28.4s, v12.4s, v7.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v29.4s, v13.4s, v7.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v30.4s, v14.4s, v7.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "fmla v31.4s, v15.4s, v7.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr s2, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x4\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr s3, [a_ptr3]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x4\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" + "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" + "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" + "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "str q25, [c_ptr2, #0x10]\n" + "str q26, [c_ptr2, #0x20]\n" + "str q27, [c_ptr2, #0x30]\n" + "str q28, [c_ptr3]\n" + "str q29, [c_ptr3, #0x10]\n" + "str q30, [c_ptr3, #0x20]\n" + "str q31, [c_ptr3, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" + ); + break; + } + + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp deleted file mode 100644 index 9b70827125..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -namespace arm_gemm { - -// Actual kernel implementations -void a64_sgemm_nativeA_pretransposeB_16x4(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int); - -// Native A/Pretranspose B SGEMM "strategy" class. -// -// This describes the characteristics of a family of kernels, in terms of -// the required interleave properties and the output block size. -// -// All kernels in the family must share these characteristics. The actual -// kernel to be used can be chosen at runtime, based on the CPUInfo -// structure. -class sgemm_nativeA_pretransposeB_16x4 { -public: - typedef float operand_type; - typedef float result_type; - - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int); - - /* Desired data layout for B buffer (used for pretranspose) */ - static const int B_interleave = 16; - static const int B_block = 1; - static const bool B_transpose = true; - - /* Kernel blocking parameters */ - static unsigned int out_width() { - return 16; - } - - static unsigned int out_height() { - return 4; - } - - static unsigned int k_unroll() { - return 1; - } - - StdTransformsFixed transforms = {}; - - // Default to the generic kernel - kern_type kernel=a64_sgemm_nativeA_pretransposeB_16x4; - - sgemm_nativeA_pretransposeB_16x4(const CPUInfo *ci) { - UNUSED(ci); - } -}; - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp deleted file mode 100644 index c51c8ff863..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp +++ /dev/null @@ -1,970 +0,0 @@ -/* - * Copyright (c) 2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __aarch64__ - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -#include -#include -#include - -#include - -namespace arm_gemm { - -void a64_sgemm_nativeA_pretransposeB_16x4(const float *A, int lda, const float *B_panel, float *C, int ldc, float beta, unsigned int numrows, unsigned int numcols, unsigned int K) { - const bool oddk = ((K % 8) >= 4); - const bool beta0 = (beta == 0.0f); - const unsigned int oddones = (K % 4); - - /* Use some small temporary arrays to cope with "ragged" M/N sizes. - * - * "dummy_A_buf" is used to avoid overreading the A input for ragged M, - * and also for output if N is not ragged. - * - * Since the B input is pretransposed it will be padded as needed, so no - * need to worry about overreading that. - * - * "C_buf" is used to avoid overreading or overwriting the output for - * ragged N cases. - */ - float dummy_A_buf[16]; - float C_buf[64]; - - std::memset(dummy_A_buf, 0, sizeof(dummy_A_buf)); - std::memset(C_buf, 0, sizeof(C_buf)); - - for (unsigned int y=0; y 1) ? 32 : 0; - const unsigned long a_incr2 = (active_rows > 2) ? 32 : 0; - const unsigned long a_incr3 = (active_rows > 3) ? 32 : 0; - - /* Starting points for A pointers on this loop */ - const float * const a_ptr0_base = A + (y * lda); - const float * const a_ptr1_base = (active_rows > 1) ? (a_ptr0_base + lda) : dummy_A_buf; - const float * const a_ptr2_base = (active_rows > 2) ? (a_ptr1_base + lda) : dummy_A_buf; - const float * const a_ptr3_base = (active_rows > 3) ? (a_ptr2_base + lda) : dummy_A_buf; - - /* Starting points for C pointers on this loop */ - float *c_ptr0 = C + (y * ldc); - float *c_ptr1 = (active_rows > 1) ? (c_ptr0 + ldc) : dummy_A_buf; - float *c_ptr2 = (active_rows > 2) ? (c_ptr1 + ldc) : dummy_A_buf; - float *c_ptr3 = (active_rows > 3) ? (c_ptr2 + ldc) : dummy_A_buf; - - for (unsigned int x0=0; x0 -#include -#include - -#include - -namespace arm_gemm { - -void a64_sgemm_native_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const int oddk = ((K % 8) >= 4) ? 1 : 0; - const int beta0 = (beta == 0.0f) ? 1 : 0; - const int oddones = (K % 4); - - float dummy_buffer[16]; - - std::memset(dummy_buffer, 0, sizeof(dummy_buffer)); - - /* For now, very naive with no blocking */ - for (int y=0; y 1) ? (a_ptr0_base + lda) : dummy_buffer; - const float * const a_ptr2_base = (activerows > 2) ? (a_ptr1_base + lda) : dummy_buffer; - const float * const a_ptr3_base = (activerows > 3) ? (a_ptr2_base + lda) : dummy_buffer; - - const unsigned long a_incr1 = (activerows > 1) ? 32 : 0; - const unsigned long a_incr2 = (activerows > 2) ? 32 : 0; - const unsigned long a_incr3 = (activerows > 3) ? 32 : 0; - - float *c_ptr0 = C + (y * ldc); - float *c_ptr1 = (activerows > 1) ? c_ptr0 + ldc : dummy_buffer; - float *c_ptr2 = (activerows > 2) ? c_ptr1 + ldc : dummy_buffer; - float *c_ptr3 = (activerows > 3) ? c_ptr2 + ldc : dummy_buffer; - - for (int x0=0; x0 transforms = {}; + + // Default to the generic kernel + kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6; + + smallK_hybrid_fp32_mla_4x6(const CPUInfo *ci) { UNUSED(ci); } +}; + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp new file mode 100644 index 0000000000..d8e8e52417 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp @@ -0,0 +1,4612 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include + +#include "arm_gemm.hpp" + + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_smallK_hybrid_fp32_mla_4x6(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; + const long ldab = lda * sizeof(float); + const long ldcb = ldc * sizeof(float); + float nullbias[4]; + if (!bias) { + memset(nullbias, 0, (4 * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y0=0; y0 transforms = {}; + + // Default to the generic kernel + kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8; + + smallK_hybrid_fp32_mla_4x8(const CPUInfo *ci) { UNUSED(ci); } +}; + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp new file mode 100644 index 0000000000..7ad52059b6 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp @@ -0,0 +1,3340 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include + +#include "arm_gemm.hpp" + + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_smallK_hybrid_fp32_mla_4x8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; + const long ldab = lda * sizeof(float); + const long ldcb = ldc * sizeof(float); + float nullbias[4]; + if (!bias) { + memset(nullbias, 0, (4 * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y0=0; y0 transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp index e8ac2d7489..6d71840801 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp @@ -25,15 +25,16 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -97,335 +98,356 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -493,364 +515,385 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" + "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ins v22.d[1], temploadreg2\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v23.d[1], temploadreg3\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -918,390 +961,408 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" - "ins v25.d[1], temploadreg1\n" - "b.eq 7f\n" - "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" - "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" - "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" - "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" - "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" - "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + "b.eq 7f\n" + "8:\n" + "str q26, [%[c_ptr0]]\n" + "subs %[loops], %[loops], #0x1\n" + "movi v26.4s, #0\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "movi v27.4s, #0\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" + "movi v28.4s, #0\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" + "movi v29.4s, #0\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" + "movi v30.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" + "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ins v23.d[1], temploadreg3\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1369,419 +1430,437 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1914,104 +1993,104 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2023,265 +2102,283 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2420,111 +2517,111 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2535,282 +2632,300 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2949,119 +3064,119 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3073,299 +3188,317 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -3511,126 +3644,126 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3641,316 +3774,334 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp index 64ab6f0fdc..d4082130ad 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp @@ -25,15 +25,16 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -93,313 +94,322 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -459,340 +469,349 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -852,364 +871,370 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1269,391 +1294,397 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1778,344 +1809,362 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q25, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2246,365 +2295,383 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2735,389 +2802,407 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q25, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -3255,410 +3340,428 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp index 6615f1f36b..02be0bd77a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_4x8 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 8; } @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp index 4ed2bc823c..c742fcc890 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp @@ -25,15 +25,16 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -114,47 +115,47 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -172,94 +173,118 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "ins v16.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" - "ins v16.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "subs %[loops], %[loops], #0x1\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr d16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "ins v16.d[1], temploadreg0\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr d16, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" - "ins v16.d[1], temploadreg0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -348,55 +373,55 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -414,23 +439,23 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -441,98 +466,122 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v16.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v17.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v17.d[1], temploadreg1\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -620,56 +669,56 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -687,32 +736,32 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -726,117 +775,141 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v17.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v18.d[1], temploadreg2\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "str q26, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v18.d[1], temploadreg2\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -925,63 +998,63 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -999,41 +1072,41 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1050,136 +1123,160 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v18.d[1], temploadreg2\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v19.d[1], temploadreg3\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v19.d[1], temploadreg3\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1334,50 +1431,50 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1397,155 +1494,179 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v19.d[1], temploadreg3\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v20.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v20.d[1], temploadreg0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1708,59 +1829,59 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1783,174 +1904,198 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v20.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v21.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v21.d[1], temploadreg1\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2113,68 +2258,68 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2200,193 +2345,217 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v21.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v22.d[1], temploadreg2\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v22.d[1], temploadreg2\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2558,77 +2727,77 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2657,212 +2826,236 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v22.d[1], temploadreg2\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v23.d[1], temploadreg3\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" "ins v22.d[1], temploadreg2\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "ldr d23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v23.d[1], temploadreg3\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp index 1eb58e1cd6..2df2c30426 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp @@ -25,15 +25,16 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -110,47 +111,47 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -168,90 +169,114 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - "movi v24.4s, #0\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "movi v24.4s, #0\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -332,55 +357,55 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -398,117 +423,141 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q17, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q17, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -588,56 +637,56 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -655,144 +704,168 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q17, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -873,63 +946,63 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -947,41 +1020,41 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -989,129 +1062,153 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q18, [%[b_ptr0], #0x20]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1258,50 +1355,50 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1310,146 +1407,170 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q19, [%[b_ptr0], #0x30]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1604,59 +1725,59 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1666,163 +1787,187 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q20, [%[b_ptr0], #0x40]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1977,68 +2122,68 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2049,180 +2194,204 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q21, [%[b_ptr0], #0x50]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2386,77 +2555,77 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2468,197 +2637,221 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr q22, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp index adeebf9bbc..1110924e5c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_4x6 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp index 8bf5debc37..ceb4a3b2d3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp @@ -25,14 +25,15 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -97,140 +98,155 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" @@ -238,114 +254,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -354,83 +367,74 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -511,369 +515,372 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" + "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ins v22.d[1], temploadreg2\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v23.d[1], temploadreg3\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -954,161 +961,175 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" @@ -1116,132 +1137,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ins v23.d[1], temploadreg3\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" @@ -1251,98 +1268,88 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1423,174 +1430,188 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" @@ -1598,141 +1619,137 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -1742,105 +1759,95 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1986,104 +1993,104 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2105,142 +2112,142 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" @@ -2252,113 +2259,113 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2510,111 +2517,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2635,152 +2642,152 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" @@ -2792,120 +2799,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3057,119 +3064,119 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3191,161 +3198,161 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" @@ -3357,128 +3364,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3637,126 +3644,126 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3777,171 +3784,171 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" @@ -3953,135 +3960,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp index 1f279afd43..4eecee4c07 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp @@ -25,14 +25,15 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -93,318 +94,309 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -477,345 +469,336 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -888,369 +871,357 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1323,396 +1294,384 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1850,104 +1809,104 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1962,120 +1921,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" @@ -2086,113 +2045,113 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2336,111 +2295,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2455,127 +2414,127 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -2586,120 +2545,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2843,119 +2802,119 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2970,135 +2929,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" @@ -3109,128 +3068,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3381,126 +3340,126 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3515,142 +3474,142 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -3661,135 +3620,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp index a260917d08..fef8909d4f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_4x8 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp index ec29698f48..f30f7e2895 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp @@ -25,14 +25,15 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -114,47 +115,47 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -172,14 +173,14 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -197,33 +198,33 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" @@ -241,32 +242,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -372,55 +373,55 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -438,23 +439,23 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -475,49 +476,49 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" @@ -530,40 +531,40 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -668,56 +669,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -735,32 +736,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -784,61 +785,61 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" "b.ne 8b\n" @@ -850,48 +851,48 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -997,63 +998,63 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -1071,41 +1072,41 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1132,72 +1133,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" "b.ne 8b\n" @@ -1209,56 +1210,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1430,50 +1431,50 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1503,83 +1504,83 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" "b.ne 8b\n" @@ -1591,64 +1592,64 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1828,59 +1829,59 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1913,94 +1914,94 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" "b.ne 8b\n" @@ -2012,72 +2013,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2257,68 +2258,68 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2354,105 +2355,105 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" "b.ne 8b\n" @@ -2464,80 +2465,80 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2726,77 +2727,77 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2835,116 +2836,116 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" "ins v22.d[1], temploadreg2\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "ldr d23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" @@ -2956,88 +2957,88 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp index 46ca013270..16c1d1740f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp @@ -25,14 +25,15 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -110,47 +111,47 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -168,14 +169,14 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -190,33 +191,33 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" @@ -233,32 +234,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -356,55 +357,55 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -422,23 +423,23 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -453,49 +454,49 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -506,40 +507,40 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -636,56 +637,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -703,32 +704,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -744,58 +745,58 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -806,48 +807,48 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -945,63 +946,63 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -1019,41 +1020,41 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1070,67 +1071,67 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1141,56 +1142,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1354,50 +1355,50 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1415,76 +1416,76 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1495,64 +1496,64 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1724,59 +1725,59 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1795,85 +1796,85 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1884,72 +1885,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2121,68 +2122,68 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2202,94 +2203,94 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -2300,80 +2301,80 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2554,77 +2555,77 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2645,103 +2646,103 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr q22, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -2752,88 +2753,88 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp index c6895a6a0e..28ef8071c2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int); +void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); class hybrid_fp16_mla_4VLx4 { @@ -40,10 +40,10 @@ public: typedef __fp16 operand_type; typedef __fp16 result_type; - typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int); + typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<__fp16>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsSVE transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_fp16_mla_4VLx4; - hybrid_fp16_mla_4VLx4(const CPUInfo *ci) - { - - } + hybrid_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp index ab41fb3743..2998f33d87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,37 +25,58 @@ #include +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 8) / 16) - 1; K -= loops_count * 16; const long regs_count = (K / 8) - 1; K -= (regs_count + 1) * 8; const long leftovers = K; + __fp16 nullbias[512]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + __fp16 minval = - static_cast<__fp16>(std::numeric_limits::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits::infinity()); + const __fp16 * const minptr = &minval; + const __fp16 * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>())); - const __fp16 *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = leftovers; const __fp16 *a_ptr0 = a_ptr0_base; const __fp16 *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(__fp16); + const __fp16 *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -69,44 +90,37 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z18.h, #0\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.h, #0\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -322,13 +336,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" - "fmla z17.h, z13.h, z4.h[1]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z4.h[1]\n" + "fmla z17.h, z13.h, z4.h[1]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z4.h[1]\n" + "fmla z18.h, z14.h, z4.h[1]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z4.h[1]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -339,9 +352,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[2]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.h, z13.h, z4.h[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z18.h, z14.h, z4.h[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z19.h, z15.h, z4.h[3]\n" "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" @@ -372,76 +387,77 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[6]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z4.h[7]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z17.h, z13.h, z4.h[7]\n" "fmla z18.h, z14.h, z4.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" "fmla z18.h, z10.h, z0.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" "fmla z18.h, z14.h, z0.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[2]\n" "fmla z18.h, z10.h, z0.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z13.h, z0.h[3]\n" "fmla z18.h, z14.h, z0.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[4]\n" "fmla z18.h, z10.h, z0.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z0.h[5]\n" "fmla z18.h, z14.h, z0.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" "fmla z18.h, z10.h, z0.h[6]\n" @@ -451,33 +467,34 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "fmla z17.h, z9.h, z0.h[2]\n" + "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z18.h, z10.h, z0.h[2]\n" "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z19.h, z11.h, z0.h[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.h, z12.h, z0.h[3]\n" - "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" @@ -504,88 +521,99 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[6]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z0.h[7]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z17.h, z13.h, z0.h[7]\n" "fmla z18.h, z14.h, z0.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[0]\n" "fmla z18.h, z10.h, z4.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z4.h[1]\n" "fmla z18.h, z14.h, z4.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[2]\n" "fmla z18.h, z10.h, z4.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z13.h, z4.h[3]\n" "fmla z18.h, z14.h, z4.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[4]\n" "fmla z18.h, z10.h, z4.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z4.h[5]\n" "fmla z18.h, z14.h, z4.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" "fmla z18.h, z10.h, z4.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -604,60 +632,49 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.h, #0\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.h, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.h, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z23.h, #0\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -974,9 +991,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z20.h, z8.h, z5.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.h, z9.h, z5.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z4.h[0]\n" @@ -1044,6 +1063,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" "fmla z21.h, z9.h, z5.h[6]\n" @@ -1060,13 +1080,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z17.h, z9.h, z0.h[0]\n" "fmla z21.h, z9.h, z1.h[0]\n" @@ -1075,12 +1094,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z17.h, z13.h, z0.h[1]\n" "fmla z21.h, z13.h, z1.h[1]\n" @@ -1089,12 +1108,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z17.h, z9.h, z0.h[2]\n" "fmla z21.h, z9.h, z1.h[2]\n" @@ -1103,12 +1123,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z17.h, z13.h, z0.h[3]\n" "fmla z21.h, z13.h, z1.h[3]\n" @@ -1117,13 +1137,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z17.h, z9.h, z0.h[4]\n" "fmla z21.h, z9.h, z1.h[4]\n" @@ -1132,12 +1151,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z17.h, z13.h, z0.h[5]\n" "fmla z21.h, z13.h, z1.h[5]\n" @@ -1146,10 +1165,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" @@ -1163,19 +1183,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" - "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z5.h, p6/z, [a_ptr1]\n" + "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" + "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z20.h, z12.h, z1.h[1]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -1237,6 +1259,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" "fmla z21.h, z9.h, z1.h[6]\n" @@ -1253,13 +1276,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z17.h, z9.h, z4.h[0]\n" "fmla z21.h, z9.h, z5.h[0]\n" @@ -1268,12 +1290,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z17.h, z13.h, z4.h[1]\n" "fmla z21.h, z13.h, z5.h[1]\n" @@ -1282,12 +1304,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z17.h, z9.h, z4.h[2]\n" "fmla z21.h, z9.h, z5.h[2]\n" @@ -1296,12 +1319,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z17.h, z13.h, z4.h[3]\n" "fmla z21.h, z13.h, z5.h[3]\n" @@ -1310,13 +1333,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z17.h, z9.h, z4.h[4]\n" "fmla z21.h, z9.h, z5.h[4]\n" @@ -1325,12 +1347,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z17.h, z13.h, z4.h[5]\n" "fmla z21.h, z13.h, z5.h[5]\n" @@ -1339,10 +1361,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" @@ -1352,9 +1375,27 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1h z20.h, p0, [c_ptr1]\n" @@ -1364,7 +1405,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -1387,76 +1428,61 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z20.h, #0\n" + "mov z24.d, z16.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.h, #0\n" + "mov z26.d, z18.d\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.h, #0\n" + "mov z27.d, z19.d\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z24.h, #0\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z25.h, #0\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z26.h, #0\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.h, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1872,10 +1898,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n" "fmla z20.h, z8.h, z5.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.h, z8.h, z6.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" @@ -1968,6 +1997,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" @@ -1992,13 +2022,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z24.h, z8.h, z2.h[0]\n" "fmla z17.h, z9.h, z0.h[0]\n" @@ -2011,12 +2040,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z24.h, z12.h, z2.h[1]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -2029,12 +2058,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z24.h, z8.h, z2.h[2]\n" "fmla z17.h, z9.h, z0.h[2]\n" @@ -2047,12 +2077,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z24.h, z12.h, z2.h[3]\n" "fmla z17.h, z13.h, z0.h[3]\n" @@ -2065,13 +2095,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z24.h, z8.h, z2.h[4]\n" "fmla z17.h, z9.h, z0.h[4]\n" @@ -2084,12 +2113,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z24.h, z12.h, z2.h[5]\n" "fmla z17.h, z13.h, z0.h[5]\n" @@ -2102,10 +2131,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" @@ -2125,18 +2155,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z24.h, z8.h, z2.h[0]\n" - "ld1rqh z5.h, p6/z, [a_ptr1]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z6.h, p6/z, [a_ptr2]\n" + "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "ld1rqh z6.h, p6/z, [a_ptr2]\n" "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -2222,6 +2255,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" @@ -2246,13 +2280,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z24.h, z8.h, z6.h[0]\n" "fmla z17.h, z9.h, z4.h[0]\n" @@ -2265,12 +2298,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z24.h, z12.h, z6.h[1]\n" "fmla z17.h, z13.h, z4.h[1]\n" @@ -2283,12 +2316,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z24.h, z8.h, z6.h[2]\n" "fmla z17.h, z9.h, z4.h[2]\n" @@ -2301,12 +2335,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z24.h, z12.h, z6.h[3]\n" "fmla z17.h, z13.h, z4.h[3]\n" @@ -2319,13 +2353,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z24.h, z8.h, z6.h[4]\n" "fmla z17.h, z9.h, z4.h[4]\n" @@ -2338,12 +2371,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z24.h, z12.h, z6.h[5]\n" "fmla z17.h, z13.h, z4.h[5]\n" @@ -2356,10 +2389,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" @@ -2373,13 +2407,39 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" "st1h z24.h, p0, [c_ptr2]\n" @@ -2391,7 +2451,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -2419,92 +2479,73 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "mov z20.h, #0\n" + "mov z24.d, z16.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.h, #0\n" + "mov z26.d, z18.d\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.h, #0\n" + "mov z27.d, z19.d\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.h, #0\n" + "mov z28.d, z16.d\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z25.h, #0\n" + "mov z29.d, z17.d\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z26.h, #0\n" + "mov z30.d, z18.d\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z27.h, #0\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.h, #0\n" + "mov z31.d, z19.d\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.h, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.h, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.h, #0\n" "add a_ptr3, a_ptr3, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" "ld1h z28.h, p0/z, [c_ptr3]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z28.h, p7/m, z28.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z29.h, p7/m, z29.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z30.h, p7/m, z30.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z31.h, p7/m, z31.h, z15.h\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -3021,11 +3062,15 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n" "fmla z20.h, z8.h, z5.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.h, z8.h, z6.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z28.h, z8.h, z7.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "fmla z29.h, z9.h, z7.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3141,6 +3186,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z31.h, z15.h, z7.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" "fmla z28.h, z8.h, z7.h[6]\n" @@ -3173,13 +3219,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z24.h, z8.h, z2.h[0]\n" "fmla z28.h, z8.h, z3.h[0]\n" @@ -3196,12 +3241,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[0]\n" "fmla z31.h, z11.h, z3.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z24.h, z12.h, z2.h[1]\n" "fmla z28.h, z12.h, z3.h[1]\n" @@ -3218,12 +3263,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[1]\n" "fmla z31.h, z15.h, z3.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z24.h, z8.h, z2.h[2]\n" "fmla z28.h, z8.h, z3.h[2]\n" @@ -3240,12 +3286,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[2]\n" "fmla z31.h, z11.h, z3.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z24.h, z12.h, z2.h[3]\n" "fmla z28.h, z12.h, z3.h[3]\n" @@ -3262,13 +3308,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[3]\n" "fmla z31.h, z15.h, z3.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z24.h, z8.h, z2.h[4]\n" "fmla z28.h, z8.h, z3.h[4]\n" @@ -3285,12 +3330,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[4]\n" "fmla z31.h, z11.h, z3.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z24.h, z12.h, z2.h[5]\n" "fmla z28.h, z12.h, z3.h[5]\n" @@ -3307,10 +3352,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[5]\n" "fmla z31.h, z15.h, z3.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" @@ -3336,17 +3382,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z28.h, z8.h, z3.h[0]\n" - "ld1rqh z6.h, p6/z, [a_ptr2]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z7.h, p6/z, [a_ptr3]\n" + "ld1rqh z6.h, p6/z, [a_ptr2]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "ld1rqh z7.h, p6/z, [a_ptr3]\n" "fmla z25.h, z9.h, z2.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.h, z10.h, z3.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" @@ -3456,6 +3506,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z31.h, z15.h, z3.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" "fmla z28.h, z8.h, z3.h[6]\n" @@ -3488,13 +3539,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[7]\n" "fmla z31.h, z15.h, z3.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z24.h, z8.h, z6.h[0]\n" "fmla z28.h, z8.h, z7.h[0]\n" @@ -3511,12 +3561,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[0]\n" "fmla z31.h, z11.h, z7.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z24.h, z12.h, z6.h[1]\n" "fmla z28.h, z12.h, z7.h[1]\n" @@ -3533,12 +3583,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[1]\n" "fmla z31.h, z15.h, z7.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z24.h, z8.h, z6.h[2]\n" "fmla z28.h, z8.h, z7.h[2]\n" @@ -3555,12 +3606,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[2]\n" "fmla z31.h, z11.h, z7.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z24.h, z12.h, z6.h[3]\n" "fmla z28.h, z12.h, z7.h[3]\n" @@ -3577,13 +3628,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[3]\n" "fmla z31.h, z15.h, z7.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z24.h, z8.h, z6.h[4]\n" "fmla z28.h, z8.h, z7.h[4]\n" @@ -3600,12 +3650,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[4]\n" "fmla z31.h, z11.h, z7.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z24.h, z12.h, z6.h[5]\n" "fmla z28.h, z12.h, z7.h[5]\n" @@ -3622,10 +3672,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[5]\n" "fmla z31.h, z15.h, z7.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" @@ -3643,16 +3694,50 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[6]\n" "fmla z31.h, z11.h, z7.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmax z28.h, p7/m, z28.h, z14.h\n" + "fmax z29.h, p7/m, z29.h, z14.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" + "fmax z30.h, p7/m, z30.h, z14.h\n" + "fmin z28.h, p7/m, z28.h, z15.h\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.h, p7/m, z29.h, z15.h\n" + "fmax z31.h, p7/m, z31.h, z14.h\n" + "fmin z30.h, p7/m, z30.h, z15.h\n" "st1h z24.h, p0, [c_ptr2]\n" + "fmin z31.h, p7/m, z31.h, z15.h\n" "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n" "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n" "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n" @@ -3667,11 +3752,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp index 76f452d963..8e3c17917b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, float, int, int, int); +void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); class hybrid_fp32_mla_4VLx4 { @@ -40,10 +40,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsSVE transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_fp32_mla_4VLx4; - hybrid_fp32_mla_4VLx4(const CPUInfo *ci) - { - - } + hybrid_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp index 1b2d7852a8..855d27a151 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,37 +25,58 @@ #include +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long leftovers = K; + float nullbias[256]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length() * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length())); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = leftovers; const float *a_ptr0 = a_ptr0_base; const float *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -69,269 +90,283 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.s, #0\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "b 2f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "2:\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "fmla z17.s, z13.s, z4.s[3]\n" + "fmla z18.s, z14.s, z4.s[3]\n" + "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "fmla z18.s, z14.s, z4.s[3]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" + "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.s, z13.s, z4.s[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z18.s, z14.s, z4.s[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z19.s, z15.s, z4.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z19.s, z11.s, z0.s[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.s, z12.s, z0.s[3]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -350,83 +385,74 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.s, #0\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z23.s, #0\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z20.s, z12.s, z1.s[1]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z21.s, z13.s, z1.s[1]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -440,12 +466,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z1.s[2]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z22.s, z10.s, z1.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" @@ -491,12 +517,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z5.s[2]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z4.s[2]\n" + "fmla z22.s, z10.s, z5.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" @@ -510,19 +536,19 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -545,12 +571,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z1.s[2]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z22.s, z10.s, z1.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" @@ -571,9 +597,11 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.s, z9.s, z5.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z22.s, z10.s, z5.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" @@ -592,6 +620,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" @@ -607,14 +636,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z5.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z17.s, z9.s, z0.s[0]\n" "fmla z21.s, z9.s, z1.s[0]\n" @@ -622,13 +650,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z1.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" @@ -636,11 +664,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z1.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" @@ -649,23 +678,26 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z1.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z21.s, z9.s, z1.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z20.s, z12.s, z1.s[1]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" @@ -678,6 +710,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" @@ -693,14 +726,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z1.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" @@ -708,13 +740,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z5.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" @@ -722,11 +754,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z5.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" @@ -735,10 +768,28 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1w z20.s, p0, [c_ptr1]\n" @@ -748,7 +799,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -760,110 +811,97 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "c_ptr2 .req X3\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" "whilelt p1.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z24.d, z16.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.s, #0\n" + "mov z26.d, z18.d\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.s, #0\n" + "mov z27.d, z19.d\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z24.s, #0\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z25.s, #0\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z26.s, #0\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" @@ -887,8 +925,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z21.s, z9.s, z1.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -955,8 +993,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z21.s, z9.s, z5.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" @@ -982,19 +1020,19 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" @@ -1026,8 +1064,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z21.s, z9.s, z1.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -1059,10 +1097,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.s, z8.s, z6.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" @@ -1090,6 +1131,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z27.s, z15.s, z6.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" @@ -1113,14 +1155,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z24.s, z8.s, z2.s[0]\n" "fmla z17.s, z9.s, z0.s[0]\n" @@ -1132,13 +1173,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z17.s, z13.s, z0.s[1]\n" @@ -1150,11 +1191,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" @@ -1167,24 +1209,28 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1rqw z6.s, p6/z, [a_ptr2]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1205,6 +1251,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z27.s, z15.s, z2.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" @@ -1228,14 +1275,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z17.s, z9.s, z4.s[0]\n" @@ -1247,13 +1293,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z17.s, z13.s, z4.s[1]\n" @@ -1265,11 +1311,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" @@ -1282,14 +1329,40 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" "st1w z24.s, p0, [c_ptr2]\n" @@ -1301,7 +1374,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1316,132 +1389,115 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "c_ptr3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" "whilelt p1.s, %[temp], %[width]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" "incw %[temp], all, mul #1\n" "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "mov z20.s, #0\n" + "mov z24.d, z16.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.s, #0\n" + "mov z26.d, z18.d\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.s, #0\n" + "mov z27.d, z19.d\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.s, #0\n" + "mov z28.d, z16.d\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z25.s, #0\n" + "mov z29.d, z17.d\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z26.s, #0\n" + "mov z30.d, z18.d\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z27.s, #0\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "mov z28.s, #0\n" + "mov z31.d, z19.d\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z28.s, p7/m, z28.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z29.s, p7/m, z29.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p7/z, [a_ptr3]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "add a_ptr3, a_ptr3, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" @@ -1471,8 +1527,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1556,8 +1612,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "fmla z29.s, z9.s, z7.s[2]\n" @@ -1591,21 +1647,21 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p7/z, [a_ptr3]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1644,8 +1700,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1686,11 +1742,15 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.s, z8.s, z6.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z28.s, z8.s, z7.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1725,6 +1785,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z31.s, z15.s, z7.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" @@ -1756,14 +1817,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z24.s, z8.s, z2.s[0]\n" "fmla z28.s, z8.s, z3.s[0]\n" @@ -1779,13 +1839,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" @@ -1801,11 +1861,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" "fmla z31.s, z15.s, z3.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" @@ -1822,25 +1883,30 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" "fmla z31.s, z11.s, z3.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p6/z, [a_ptr2]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p6/z, [a_ptr3]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z6.s, p6/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1rqw z7.s, p6/z, [a_ptr3]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1869,6 +1935,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z31.s, z15.s, z3.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" @@ -1900,14 +1967,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" "fmla z31.s, z15.s, z3.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z28.s, z8.s, z7.s[0]\n" @@ -1923,13 +1989,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" "fmla z31.s, z11.s, z7.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" @@ -1945,11 +2011,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" "fmla z31.s, z15.s, z7.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" @@ -1966,17 +2033,51 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" "fmla z31.s, z11.s, z7.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmax z28.s, p7/m, z28.s, z14.s\n" + "fmax z29.s, p7/m, z29.s, z14.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" + "fmax z30.s, p7/m, z30.s, z14.s\n" + "fmin z28.s, p7/m, z28.s, z15.s\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.s, p7/m, z29.s, z15.s\n" + "fmax z31.s, p7/m, z31.s, z14.s\n" + "fmin z30.s, p7/m, z30.s, z15.s\n" "st1w z24.s, p0, [c_ptr2]\n" + "fmin z31.s, p7/m, z31.s, z15.s\n" "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n" "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n" "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n" @@ -1991,11 +2092,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp index ffd7918b7a..d8422105cc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class hybrid_s8s32_dot_4VLx4 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_s8s32_dot_4VLx4; - hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) - { - - } + hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp index 673f186524..d57557a242 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 const unsigned long ldab = lda * sizeof(int8_t); int32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(int32_t); for (int x0=0; x0())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length())); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const int8_t *a_ptr0 = a_ptr0_base; const int8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(int32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "sdot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z18.s, z14.b, z4.b[1]\n" + "sdot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "sdot z19.s, z15.b, z4.b[1]\n" + "sdot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "sdot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "sdot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "sdot z16.s, z12.b, z0.b[1]\n" + "sdot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "sdot z17.s, z13.b, z0.b[1]\n" + "sdot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "sdot z18.s, z14.b, z0.b[1]\n" + "sdot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "sdot z19.s, z15.b, z0.b[1]\n" + "sdot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "sdot z16.s, z8.b, z0.b[2]\n" + "sdot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[2]\n" + "sdot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "sdot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z17.s, z9.b, z0.b[0]\n" "sdot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z17.s, z13.b, z0.b[1]\n" "sdot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "sdot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z17.s, z13.b, z4.b[1]\n" "sdot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "sdot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z24.s, z8.b, z2.b[0]\n" "sdot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z24.s, z12.b, z2.b[1]\n" "sdot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z24.s, z8.b, z6.b[0]\n" "sdot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z24.s, z12.b, z6.b[1]\n" "sdot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "sdot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "sdot z25.s, z9.b, z6.b[0]\n" "sdot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z24.s, z8.b, z2.b[0]\n" "sdot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z24.s, z12.b, z2.b[1]\n" "sdot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "sdot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z24.s, z8.b, z6.b[0]\n" "sdot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z24.s, z12.b, z6.b[1]\n" "sdot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp index 2e51349d35..5dab1da135 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class hybrid_u8u32_dot_4VLx4 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,20 +53,32 @@ public: return get_vector_length() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_u8u32_dot_4VLx4; - hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) - { - - } + hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp index d34d0e5fc7..bf3e8ca26a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const uint8_t *a_ptr0 = a_ptr0_base; const uint8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "udot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z14.b, z4.b[1]\n" + "udot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z19.s, z15.b, z4.b[1]\n" + "udot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "udot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "udot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "udot z17.s, z13.b, z0.b[1]\n" + "udot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "udot z18.s, z14.b, z0.b[1]\n" + "udot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "udot z19.s, z15.b, z0.b[1]\n" + "udot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "udot z16.s, z8.b, z0.b[2]\n" + "udot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "udot z17.s, z9.b, z0.b[2]\n" + "udot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "udot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "udot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp index 2ca4ce25e8..a74f9efb87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_fp16_mla_3VLx8; - interleaved_fp16_mla_3VLx8(const CPUInfo *ci) - { - - } + interleaved_fp16_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp index 8c1fe6d0b6..1e73d8e185 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_fp32_mla_3VLx8; - interleaved_fp32_mla_3VLx8(const CPUInfo *ci) - { - - } + interleaved_fp32_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp index cbb21387b1..8668ea7fbc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_s8s32_dot_3VLx8; - interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) - { - - } + interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp index 99c039e121..d3c8851154 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Arm Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_u8u32_dot_3VLx8; - interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) - { - - } + interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp index 6cce601dcc..741f200d25 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,11 @@ #ifdef __ARM_FEATURE_SVE - - - namespace arm_gemm { // Actual kernel implementations -void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int); +void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); class native_fp16_mla_4VLx4 { @@ -40,10 +37,10 @@ public: typedef __fp16 operand_type; typedef __fp16 result_type; - typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int); + typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +50,32 @@ public: return get_vector_length<__fp16>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + // Default to the generic kernel kern_type kernel=sve_native_fp16_mla_4VLx4; - native_fp16_mla_4VLx4(const CPUInfo *ci) - { - - } + native_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp index f1aaeb13ee..14dd38bd25 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,30 +25,49 @@ #include +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) { const long loops_count = ((K + 8) / 16) - 1; K -= loops_count * 16; const long regs_count = (K / 8) - 1; K -= (regs_count + 1) * 8; const long leftovers = K; + __fp16 nullbias[512]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + __fp16 minval = - static_cast<__fp16>(std::numeric_limits::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits::infinity()); + const __fp16 * const minptr = &minval; + const __fp16 * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>())); - const __fp16 *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -56,6 +75,8 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld const __fp16 *a_ptr0 = a_ptr0_base; const __fp16 *b_ptr0 = B + x0; long ldbb = ldb * sizeof(__fp16); + const unsigned long ldcb = ldc * sizeof(__fp16); + const __fp16 *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -64,52 +85,34 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z18.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "whilelt p2.h, %[temp], %[width]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" + "inch %[temp], all, mul #1\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "cbz %[loops], 2f\n" - "3:\n" + "whilelt p3.h, %[temp], %[width]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" + "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" "fmla z18.h, z10.h, z0.h[0]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "subs %[loops], %[loops], #0x1\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -119,70 +122,69 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" - "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[3]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[3]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[3]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[4]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z9.h, z0.h[4]\n" + "fmla z16.h, z8.h, z0.h[4]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[4]\n" + "fmla z17.h, z9.h, z0.h[4]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[4]\n" + "fmla z18.h, z10.h, z0.h[4]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[4]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[5]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[5]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[5]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[6]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z9.h, z0.h[6]\n" + "fmla z16.h, z8.h, z0.h[6]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[6]\n" + "fmla z17.h, z9.h, z0.h[6]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[6]\n" + "fmla z18.h, z10.h, z0.h[6]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[6]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[7]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[7]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[7]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[7]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.h, z9.h, z4.h[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z4.h[0]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z4.h[0]\n" + "fmla z18.h, z10.h, z4.h[0]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.h, z11.h, z4.h[0]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" @@ -247,94 +249,95 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z18.h, z14.h, z4.h[7]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z15.h, z4.h[7]\n" + "b.ne 2b\n" + "1:\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "b.ne 3b\n" - "2:\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[3]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[3]\n" + "fmla z16.h, z12.h, z0.h[3]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[4]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[4]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[4]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[4]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[5]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[5]\n" + "fmla z16.h, z12.h, z0.h[5]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[5]\n" + "fmla z17.h, z13.h, z0.h[5]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[5]\n" + "fmla z18.h, z14.h, z0.h[5]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[5]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[6]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[6]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[6]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[6]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[7]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[7]\n" + "fmla z16.h, z12.h, z0.h[7]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[7]\n" + "fmla z17.h, z13.h, z0.h[7]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[7]\n" + "fmla z18.h, z14.h, z0.h[7]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[7]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z4.h[0]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.h, z9.h, z4.h[0]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.h, z10.h, z4.h[0]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z19.h, z11.h, z4.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" + "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z13.h, z4.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.h, z14.h, z4.h[1]\n" @@ -392,7 +395,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[7]\n" "fmla z18.h, z14.h, z4.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -403,7 +406,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[0]\n" "fmla z18.h, z10.h, z0.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -414,7 +417,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[1]\n" "fmla z18.h, z14.h, z0.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -425,7 +428,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[2]\n" "fmla z18.h, z10.h, z0.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -436,7 +439,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[3]\n" "fmla z18.h, z14.h, z0.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -447,7 +450,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[4]\n" "fmla z18.h, z10.h, z0.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -458,7 +461,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[5]\n" "fmla z18.h, z14.h, z0.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -468,73 +471,73 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[6]\n" "fmla z18.h, z10.h, z0.h[6]\n" "fmla z19.h, z11.h, z0.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[3]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[3]\n" + "fmla z16.h, z12.h, z0.h[3]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[4]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[4]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[4]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[4]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[5]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[5]\n" + "fmla z16.h, z12.h, z0.h[5]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[5]\n" + "fmla z17.h, z13.h, z0.h[5]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[5]\n" + "fmla z18.h, z14.h, z0.h[5]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[5]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[6]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[6]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[6]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[6]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.h, z12.h, z0.h[7]\n" "fmla z17.h, z13.h, z0.h[7]\n" "fmla z18.h, z14.h, z0.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -545,7 +548,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[0]\n" "fmla z18.h, z10.h, z4.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -556,7 +559,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[1]\n" "fmla z18.h, z14.h, z4.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -567,7 +570,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[2]\n" "fmla z18.h, z10.h, z4.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -578,7 +581,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[3]\n" "fmla z18.h, z14.h, z4.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -589,7 +592,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[4]\n" "fmla z18.h, z10.h, z4.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -600,7 +603,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[5]\n" "fmla z18.h, z14.h, z4.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -610,14 +613,24 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[6]\n" "fmla z18.h, z10.h, z4.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -631,66 +644,33 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z19.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.h, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z23.h, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "inch %[temp], all, mul #1\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z22.d, z18.d\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -906,23 +886,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z22.h, z10.h, z1.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" @@ -1026,9 +1006,11 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z21.h, z9.h, z5.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z23.h, z11.h, z5.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" @@ -1112,7 +1094,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1127,7 +1109,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1142,7 +1124,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1157,7 +1139,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1172,7 +1154,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1187,7 +1169,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1202,7 +1184,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1216,22 +1198,22 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[6]\n" "fmla z19.h, z11.h, z0.h[6]\n" "fmla z23.h, z11.h, z1.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z22.h, z10.h, z1.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" @@ -1239,6 +1221,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z20.h, z12.h, z1.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z13.h, z0.h[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z21.h, z13.h, z1.h[1]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z14.h, z0.h[1]\n" @@ -1315,7 +1298,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1330,7 +1313,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1345,7 +1328,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1360,7 +1343,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1375,7 +1358,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1390,7 +1373,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1405,7 +1388,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1419,10 +1402,28 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1h z20.h, p0, [c_ptr1]\n" @@ -1432,7 +1433,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -1450,82 +1451,39 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" - "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z20.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.h, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z24.h, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "mov z25.h, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z26.h, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z27.h, #0\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" - "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" - "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "inch %[temp], all, mul #1\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" + "mov z26.d, z18.d\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -1551,9 +1509,9 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z20.h, z12.h, z1.h[1]\n" "add a_ptr2, a_ptr2, #0x20\n" + "fmla z20.h, z12.h, z1.h[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.h, z12.h, z2.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -1808,23 +1766,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p7/z, [a_ptr2]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "fmla z22.h, z10.h, z1.h[0]\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -1960,10 +1918,13 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z24.h, z8.h, z6.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "fmla z26.h, z10.h, z6.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -2080,7 +2041,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2099,7 +2060,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2118,7 +2079,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2137,7 +2098,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2156,7 +2117,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2175,7 +2136,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2194,7 +2155,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2212,26 +2173,28 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[6]\n" "fmla z23.h, z11.h, z1.h[6]\n" "fmla z27.h, z11.h, z2.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p6/z, [a_ptr2]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" @@ -2344,7 +2307,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2363,7 +2326,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2382,7 +2345,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2401,7 +2364,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2420,7 +2383,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2439,7 +2402,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2458,7 +2421,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2476,14 +2439,40 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" "st1h z24.h, p0, [c_ptr2]\n" @@ -2495,7 +2484,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -2518,98 +2507,45 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z28.d, z16.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "mov z20.h, #0\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.h, #0\n" + "mov z29.d, z17.d\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.h, #0\n" + "inch %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z25.h, #0\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z26.h, #0\n" + "mov z26.d, z18.d\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z30.d, z18.d\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z27.h, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "mov z28.h, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z29.h, #0\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "mov z30.h, #0\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z31.h, #0\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" - "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" - "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" - "ld1h z28.h, p0/z, [c_ptr3]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" - "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" - "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" - "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z28.h, p7/m, z28.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z29.h, p7/m, z29.h, z15.h\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z30.h, p7/m, z30.h, z15.h\n" - "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z31.h, p7/m, z31.h, z15.h\n" - "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "mov z31.d, z19.d\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -2959,23 +2895,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z28.h, z8.h, z3.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p7/z, [a_ptr2]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z28.h, z8.h, z3.h[0]\n" "ld1rqh z7.h, p7/z, [a_ptr3]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" @@ -3143,11 +3079,15 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z20.h, z8.h, z5.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.h, z8.h, z6.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z28.h, z8.h, z7.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z29.h, z9.h, z7.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" @@ -3297,7 +3237,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3320,7 +3260,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "fmla z31.h, z11.h, z3.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3343,7 +3283,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" "fmla z31.h, z15.h, z3.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3366,7 +3306,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" "fmla z31.h, z11.h, z3.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3389,7 +3329,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" "fmla z31.h, z15.h, z3.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3412,7 +3352,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" "fmla z31.h, z11.h, z3.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3435,7 +3375,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" "fmla z31.h, z15.h, z3.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3457,27 +3397,30 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[6]\n" "fmla z27.h, z11.h, z2.h[6]\n" "fmla z31.h, z11.h, z3.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z28.h, z8.h, z3.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p6/z, [a_ptr2]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z28.h, z8.h, z3.h[0]\n" "ld1rqh z7.h, p6/z, [a_ptr3]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "fmla z25.h, z9.h, z2.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.h, z10.h, z3.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" @@ -3622,7 +3565,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" "fmla z31.h, z15.h, z3.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3645,7 +3588,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" "fmla z31.h, z11.h, z7.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3668,7 +3611,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" "fmla z31.h, z15.h, z7.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3691,7 +3634,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" "fmla z31.h, z11.h, z7.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3714,7 +3657,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" "fmla z31.h, z15.h, z7.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3737,7 +3680,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" "fmla z31.h, z11.h, z7.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3760,7 +3703,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" "fmla z31.h, z15.h, z7.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3782,17 +3725,51 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" "fmla z31.h, z11.h, z7.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmax z28.h, p7/m, z28.h, z14.h\n" + "fmax z29.h, p7/m, z29.h, z14.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" + "fmax z30.h, p7/m, z30.h, z14.h\n" + "fmin z28.h, p7/m, z28.h, z15.h\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.h, p7/m, z29.h, z15.h\n" + "fmax z31.h, p7/m, z31.h, z14.h\n" + "fmin z30.h, p7/m, z30.h, z15.h\n" "st1h z24.h, p0, [c_ptr2]\n" + "fmin z31.h, p7/m, z31.h, z15.h\n" "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n" "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n" "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n" @@ -3807,11 +3784,12 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp index d7f9f20074..19e5fbd974 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,13 +25,11 @@ #ifdef __ARM_FEATURE_SVE - - namespace arm_gemm { // Actual kernel implementations -void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int); +void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); class native_fp32_mla_4VLx4 { @@ -39,10 +37,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -52,20 +50,32 @@ public: return get_vector_length() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + // Default to the generic kernel kern_type kernel=sve_native_fp32_mla_4VLx4; - native_fp32_mla_4VLx4(const CPUInfo *ci) - { - - } + native_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp index 6e225669fc..3fc0e5fa36 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,30 +25,49 @@ #include +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long leftovers = K; + float nullbias[256]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length() * sizeof(float))); + } + float minval = - static_cast(std::numeric_limits::infinity()); + float maxval = static_cast(std::numeric_limits::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length())); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -56,6 +75,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, const float *a_ptr0 = a_ptr0_base; const float *b_ptr0 = B + x0; long ldbb = ldb * sizeof(float); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -64,191 +85,173 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "whilelt p2.s, %[temp], %[width]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" + "incw %[temp], all, mul #1\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "cbz %[loops], 1f\n" "2:\n" - "cbz %[loops], 3f\n" - "4:\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z16.s, z12.s, z0.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z19.s, z15.s, z0.s[1]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "fmla z18.s, z14.s, z0.s[3]\n" - "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z14.s, z0.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "fmla z18.s, z10.s, z4.s[0]\n" - "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" - "fmla z18.s, z14.s, z4.s[1]\n" - "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z4.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" - "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z4.s[3]\n" - "fmla z17.s, z13.s, z4.s[3]\n" - "fmla z18.s, z14.s, z4.s[3]\n" - "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z4.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z4.s[3]\n" + "b.ne 2b\n" + "1:\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "b.ne 4b\n" - "3:\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z18.s, z10.s, z0.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.s, z11.s, z0.s[0]\n" - "fmla z16.s, z12.s, z0.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" - "fmla z18.s, z14.s, z0.s[1]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z15.s, z0.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z19.s, z15.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" - "fmla z18.s, z14.s, z0.s[3]\n" - "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" - "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" "fmla z17.s, z13.s, z4.s[3]\n" "fmla z18.s, z14.s, z4.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -259,7 +262,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -270,7 +273,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z13.s, z0.s[1]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -280,37 +283,37 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z18.s, z10.s, z0.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.s, z11.s, z0.s[0]\n" - "fmla z16.s, z12.s, z0.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" - "fmla z18.s, z14.s, z0.s[1]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z15.s, z0.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z19.s, z15.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.s, z12.s, z0.s[3]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -321,7 +324,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z4.s[0]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -332,7 +335,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z13.s, z4.s[1]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -342,14 +345,24 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z4.s[2]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -363,61 +376,33 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z23.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" + "incw %[temp], all, mul #1\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z22.d, z18.d\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "2:\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "cbz %[loops], 3f\n" - "4:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -425,200 +410,202 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z23.s, z11.s, z1.s[0]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z16.s, z12.s, z0.s[1]\n" + "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z20.s, z12.s, z1.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z13.s, z0.s[1]\n" + "fmla z20.s, z12.s, z1.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z17.s, z13.s, z0.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z22.s, z14.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[2]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z20.s, z8.s, z1.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z12.s, z0.s[3]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z12.s, z1.s[3]\n" + "fmla z16.s, z12.s, z0.s[3]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z20.s, z12.s, z1.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z22.s, z14.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z8.s, z4.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z16.s, z8.s, z4.s[0]\n" + "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z22.s, z10.s, z5.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z22.s, z14.s, z5.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z5.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z22.s, z10.s, z5.s[2]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[3]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[3]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z5.s[3]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[3]\n" "fmla z22.s, z14.s, z5.s[3]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "b.ne 4b\n" - "3:\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z22.s, z14.s, z1.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z22.s, z14.s, z1.s[3]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z22.s, z10.s, z5.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z23.s, z11.s, z5.s[0]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z22.s, z14.s, z5.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z20.s, z8.s, z5.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" @@ -631,7 +618,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z5.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -646,7 +633,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z1.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -661,7 +648,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z1.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -675,40 +662,41 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z1.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z21.s, z13.s, z1.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z22.s, z14.s, z1.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" @@ -722,7 +710,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z1.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -737,7 +725,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z5.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -752,7 +740,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z5.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -766,10 +754,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1w z20.s, p0, [c_ptr1]\n" @@ -779,7 +785,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -791,85 +797,45 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "c_ptr2 .req X3\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.s, #0\n" + "incw %[temp], all, mul #1\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z24.s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "mov z25.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z26.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z27.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" + "mov z26.d, z18.d\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "2:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -879,28 +845,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z26.s, z10.s, z2.s[0]\n" + "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[0]\n" + "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z20.s, z12.s, z1.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z12.s, z2.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -916,9 +882,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -933,9 +899,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" @@ -953,12 +919,12 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" "fmla z25.s, z9.s, z6.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[0]\n" "fmla z22.s, z10.s, z5.s[0]\n" "fmla z26.s, z10.s, z6.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -970,9 +936,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" @@ -987,9 +953,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z6.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" @@ -1004,9 +970,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[3]\n" "fmla z24.s, z12.s, z6.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[3]\n" "fmla z21.s, z13.s, z5.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[3]\n" @@ -1016,37 +982,37 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z22.s, z10.s, z1.s[0]\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -1061,9 +1027,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -1078,9 +1044,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" @@ -1098,12 +1064,15 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z22.s, z10.s, z5.s[0]\n" "fmla z26.s, z10.s, z6.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -1115,9 +1084,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" @@ -1152,7 +1121,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1171,7 +1140,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1190,7 +1159,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1208,36 +1177,38 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -1272,7 +1243,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1291,7 +1262,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1310,7 +1281,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1328,14 +1299,40 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" "st1w z24.s, p0, [c_ptr2]\n" @@ -1347,7 +1344,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1362,103 +1359,53 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "c_ptr3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "ptrue p7.s\n" - "whilelt p1.s, %[temp], %[width]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" + "whilelt p6.s, %[temp], %[leftovers]\n" + "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[temp], %[width]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z28.d, z16.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "mov z20.s, #0\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.s, #0\n" + "mov z29.d, z17.d\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.s, #0\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z25.s, #0\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z26.s, #0\n" + "mov z26.d, z18.d\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z30.d, z18.d\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z27.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "mov z28.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z28.s, p7/m, z28.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z29.s, p7/m, z29.s, z15.s\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z30.s, p7/m, z30.s, z15.s\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmul z31.s, p7/m, z31.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "2:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "mov z31.d, z19.d\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -1470,23 +1417,23 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z29.s, z9.s, z3.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z22.s, z10.s, z1.s[0]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z22.s, z10.s, z1.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr3, a_ptr3, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1495,8 +1442,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1516,8 +1463,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1537,8 +1484,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" "fmla z28.s, z12.s, z3.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" "fmla z25.s, z13.s, z2.s[3]\n" "fmla z29.s, z13.s, z3.s[3]\n" @@ -1562,9 +1509,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z28.s, z8.s, z7.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z6.s[0]\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1583,8 +1530,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" "fmla z25.s, z13.s, z6.s[1]\n" "fmla z29.s, z13.s, z7.s[1]\n" @@ -1604,8 +1551,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "fmla z29.s, z9.s, z7.s[2]\n" @@ -1625,8 +1572,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[3]\n" "fmla z24.s, z12.s, z6.s[3]\n" "fmla z28.s, z12.s, z7.s[3]\n" - "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[3]\n" "fmla z21.s, z13.s, z5.s[3]\n" "fmla z25.s, z13.s, z6.s[3]\n" "fmla z29.s, z13.s, z7.s[3]\n" @@ -1640,28 +1587,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z28.s, z8.s, z3.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z28.s, z8.s, z3.s[0]\n" "ld1rqw z7.s, p7/z, [a_ptr3]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "fmla z29.s, z9.s, z3.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z22.s, z10.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1674,8 +1621,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1695,8 +1642,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1716,8 +1663,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" "fmla z28.s, z12.s, z3.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" "fmla z25.s, z13.s, z2.s[3]\n" "fmla z29.s, z13.s, z3.s[3]\n" @@ -1740,11 +1687,15 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z28.s, z8.s, z7.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" @@ -1762,8 +1713,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" "fmla z25.s, z13.s, z6.s[1]\n" "fmla z29.s, z13.s, z7.s[1]\n" @@ -1810,7 +1761,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1833,7 +1784,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1856,7 +1807,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" "fmla z31.s, z15.s, z3.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1878,27 +1829,30 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" "fmla z31.s, z11.s, z3.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z28.s, z8.s, z3.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z28.s, z8.s, z3.s[0]\n" "ld1rqw z7.s, p6/z, [a_ptr3]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.s, z9.s, z3.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.s, z10.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1911,8 +1865,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1959,7 +1913,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" "fmla z31.s, z15.s, z3.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1982,7 +1936,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" "fmla z31.s, z11.s, z7.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -2005,7 +1959,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" "fmla z31.s, z15.s, z7.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2027,17 +1981,51 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" "fmla z31.s, z11.s, z7.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmax z28.s, p7/m, z28.s, z14.s\n" + "fmax z29.s, p7/m, z29.s, z14.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" + "fmax z30.s, p7/m, z30.s, z14.s\n" + "fmin z28.s, p7/m, z28.s, z15.s\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.s, p7/m, z29.s, z15.s\n" + "fmax z31.s, p7/m, z31.s, z14.s\n" + "fmin z30.s, p7/m, z30.s, z15.s\n" "st1w z24.s, p0, [c_ptr2]\n" + "fmin z31.s, p7/m, z31.s, z15.s\n" "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n" "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n" "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n" @@ -2052,11 +2040,12 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp index 8b98358cd4..1b9d1312b5 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,11 +27,12 @@ #include + namespace arm_gemm { // Actual kernel implementations -void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int); +void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class native_s8s32_dot_4VLx4 { @@ -39,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -52,20 +53,32 @@ public: return get_vector_length() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + // Default to the generic kernel kern_type kernel=sve_native_s8s32_dot_4VLx4; - native_s8s32_dot_4VLx4(const CPUInfo *ci) - { - - } + native_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp index abee1bbe1f..95cf88ab54 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp @@ -25,14 +25,14 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; const long regs_count = (K / 16) - 1; @@ -46,11 +46,9 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l const unsigned long ldab = lda * sizeof(int8_t); int32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(int32_t); for (int x0=0; x0())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length())); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -62,95 +60,68 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l const int8_t *b_ptr2 = b_ptr1 + ldb; const int8_t *b_ptr3 = b_ptr2 + ldb; long ldbb = ldb * sizeof(int8_t) * 4; + const unsigned long ldcb = ldc * sizeof(int32_t); switch(M-y) { case 1: __asm __volatile ( + "mov z16.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z17.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z18.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z19.s, #0\n" "incw %[temp], all, mul #1\n" "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z19.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "whilelt p1.s, %[temp], %[width]\n" + "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip2 z11.b, z8.b, z9.b\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" + "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "whilelt p3.s, %[temp], %[width]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z10.b, z11.b, z12.b\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "sdot z16.s, z8.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "zip2 z15.b, z12.b, z13.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "zip2 z15.b, z12.b, z13.b\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z12.b, z13.b, z14.b\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" @@ -289,32 +260,32 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "b.ne 3b\n" - "2:\n" "zip1 z10.b, z10.b, z8.b\n" "zip1 z8.b, z9.b, z10.b\n" + "b.ne 2b\n" + "1:\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" @@ -377,6 +348,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z10.b, z10.b, z8.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z8.b, z9.b, z10.b\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" @@ -437,7 +409,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -459,7 +431,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -481,7 +453,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -502,31 +474,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -540,33 +512,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -580,33 +552,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -620,33 +592,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -660,36 +632,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z16.s, z12.b, z0.b[1]\n" + "sdot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z13.b, z0.b[1]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "sdot z16.s, z12.b, z0.b[1]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "sdot z17.s, z13.b, z0.b[1]\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z14.b, z0.b[1]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z15.b, z0.b[1]\n" @@ -718,7 +690,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -740,7 +712,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -762,7 +734,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -783,31 +755,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -821,33 +793,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -861,33 +833,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -901,33 +873,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -941,14 +913,14 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -956,125 +928,81 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" + "mov z17.s, #0\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z18.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z19.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z20.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z21.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z20.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z21.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p1.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ptrue p7.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z22.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z23.s, #0\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "ld1rqb z1.b, p7/z, [a_ptr1]\n" "zip1 z10.b, z10.b, z8.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "zip1 z10.b, z11.b, z12.b\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" "subs %[loops], %[loops], #0x1\n" - "sdot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -1244,38 +1172,38 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "sdot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1357,23 +1285,25 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z20.s, z8.b, z5.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z14.b, z14.b, z12.b\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z4.b[0]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z18.s, z10.b, z4.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z23.s, z11.b, z5.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "sdot z23.s, z11.b, z5.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1420,7 +1350,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1446,7 +1376,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1472,7 +1402,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1497,31 +1427,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1539,33 +1469,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1583,33 +1513,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1627,33 +1557,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1671,35 +1601,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "sdot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1746,7 +1677,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1772,7 +1703,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1798,7 +1729,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1823,31 +1754,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1865,33 +1796,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1909,33 +1840,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1953,33 +1884,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1997,7 +1928,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -2010,7 +1941,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -2020,119 +1951,65 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z17.s, #0\n" "add a_ptr2, a_ptr1, %[lda]\n" + "mov z18.s, #0\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z19.s, #0\n" "add c_ptr2, c_ptr1, %[ldc]\n" + "mov z20.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z21.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z22.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z23.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z24.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z21.s, #0\n" + "mov z26.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z22.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "incw %[temp], all, mul #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z23.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z24.s, #0\n" + "ptrue p7.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "whilelt p2.s, %[temp], %[width]\n" "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "mov z25.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "mov z26.s, #0\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "mov z27.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z8.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z9.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "zip2 z11.b, z11.b, z12.b\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "sdot z24.s, z8.b, z2.b[0]\n" @@ -2361,24 +2238,23 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" @@ -2386,12 +2262,13 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -2491,8 +2368,11 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip1 z14.b, z14.b, z12.b\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -2571,7 +2451,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2601,7 +2481,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -2631,7 +2511,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2660,31 +2540,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2706,33 +2586,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2754,33 +2634,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2802,33 +2682,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2850,34 +2730,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -2942,7 +2824,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2972,7 +2854,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3002,7 +2884,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3031,31 +2913,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3077,33 +2959,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3125,33 +3007,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3173,33 +3055,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3221,7 +3103,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -3240,7 +3122,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -3253,175 +3135,109 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "whilelt p6.b, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "whilelt p4.b, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" + "add a_ptr3, a_ptr2, %[lda]\n" "mov z19.s, #0\n" - "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "mov z20.s, #0\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "mov z21.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "mov z22.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "whilelt p6.b, %[temp], %[leftovers]\n" "mov z23.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "whilelt p0.s, %[temp], %[width]\n" "mov z24.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "whilelt p4.b, %[temp], %[width]\n" "mov z25.s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" - "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "incw %[temp], all, mul #1\n" "mov z26.s, #0\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "mov z27.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip2 z9.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "mov z28.s, #0\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "mov z29.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" + "zip2 z11.b, z8.b, z9.b\n" + "incw %[temp], all, mul #1\n" + "zip1 z9.b, z8.b, z9.b\n" + "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "mov z30.s, #0\n" - "zip2 z15.b, z12.b, z13.b\n" - "zip1 z13.b, z12.b, z13.b\n" + "ptrue p7.b\n" "mov z31.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "zip2 z12.b, z10.b, z8.b\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "incw %[temp], all, mul #1\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "zip1 z8.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip2 z9.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z11.b, z11.b, z12.b\n" "add a_ptr3, a_ptr3, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1rqb z6.b, p7/z, [a_ptr2]\n" "sdot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z7.b, p7/z, [a_ptr3]\n" "sdot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p7/z, [a_ptr2]\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z8.b, z14.b, z12.b\n" - "ld1rqb z7.b, p7/z, [a_ptr3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "add a_ptr3, a_ptr3, #0x20\n" "sdot z26.s, z10.b, z2.b[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -3649,29 +3465,29 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z28.s, z8.b, z3.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "sdot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p7/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3797,9 +3613,13 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z28.s, z8.b, z7.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr3, a_ptr3, #2\n" "zip1 z14.b, z14.b, z12.b\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -3893,7 +3713,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3927,7 +3747,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3961,7 +3781,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3994,31 +3814,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4044,33 +3864,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4096,33 +3916,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4148,33 +3968,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4200,37 +4020,40 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z28.s, z8.b, z3.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "sdot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p6/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" @@ -4309,7 +4132,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4343,7 +4166,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -4377,7 +4200,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4410,31 +4233,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4460,33 +4283,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4512,33 +4335,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4564,33 +4387,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4616,7 +4439,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -4641,11 +4464,12 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp index bcbd3d35f5..33e3ac6c23 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int); +void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class native_u8u32_dot_4VLx4 { @@ -40,10 +40,10 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + // Default to the generic kernel kern_type kernel=sve_native_u8u32_dot_4VLx4; - native_u8u32_dot_4VLx4(const CPUInfo *ci) - { - - } + native_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp index cdcea59c5e..994d608bf6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp @@ -25,14 +25,14 @@ #include +#include "arm_gemm.hpp" #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; const long regs_count = (K / 16) - 1; @@ -46,11 +46,9 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -62,95 +60,68 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int const uint8_t *b_ptr2 = b_ptr1 + ldb; const uint8_t *b_ptr3 = b_ptr2 + ldb; long ldbb = ldb * sizeof(uint8_t) * 4; + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: __asm __volatile ( + "mov z16.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z17.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z18.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z19.s, #0\n" "incw %[temp], all, mul #1\n" "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z19.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "whilelt p1.s, %[temp], %[width]\n" + "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip2 z11.b, z8.b, z9.b\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" + "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "whilelt p3.s, %[temp], %[width]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z10.b, z11.b, z12.b\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "udot z16.s, z8.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "zip2 z15.b, z12.b, z13.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "zip2 z15.b, z12.b, z13.b\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "udot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z12.b, z13.b, z14.b\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" @@ -289,32 +260,32 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "b.ne 3b\n" - "2:\n" "zip1 z10.b, z10.b, z8.b\n" "zip1 z8.b, z9.b, z10.b\n" + "b.ne 2b\n" + "1:\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" @@ -377,6 +348,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z10.b, z10.b, z8.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z8.b, z9.b, z10.b\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" @@ -437,7 +409,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -459,7 +431,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -481,7 +453,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -502,31 +474,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -540,33 +512,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -580,33 +552,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -620,33 +592,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -660,36 +632,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z17.s, z9.b, z0.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z13.b, z0.b[1]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "udot z16.s, z12.b, z0.b[1]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "udot z17.s, z13.b, z0.b[1]\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z14.b, z0.b[1]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z15.b, z0.b[1]\n" @@ -718,7 +690,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -740,7 +712,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -762,7 +734,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -783,31 +755,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -821,33 +793,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -861,33 +833,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -901,33 +873,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -941,14 +913,14 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -956,125 +928,81 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" + "mov z17.s, #0\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z18.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z19.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z20.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z21.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z20.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z21.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p1.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ptrue p7.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z22.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z23.s, #0\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "ld1rqb z1.b, p7/z, [a_ptr1]\n" "zip1 z10.b, z10.b, z8.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "zip1 z10.b, z11.b, z12.b\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" "subs %[loops], %[loops], #0x1\n" - "udot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -1244,38 +1172,38 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "udot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1357,23 +1285,25 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z20.s, z8.b, z5.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z14.b, z14.b, z12.b\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z4.b[0]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z18.s, z10.b, z4.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z4.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z23.s, z11.b, z5.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "udot z23.s, z11.b, z5.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1420,7 +1350,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1446,7 +1376,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1472,7 +1402,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1497,31 +1427,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1539,33 +1469,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1583,33 +1513,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1627,33 +1557,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1671,35 +1601,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "udot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1746,7 +1677,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1772,7 +1703,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1798,7 +1729,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1823,31 +1754,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1865,33 +1796,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1909,33 +1840,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1953,33 +1884,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1997,7 +1928,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -2010,7 +1941,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -2020,119 +1951,65 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z17.s, #0\n" "add a_ptr2, a_ptr1, %[lda]\n" + "mov z18.s, #0\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z19.s, #0\n" "add c_ptr2, c_ptr1, %[ldc]\n" + "mov z20.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z21.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z22.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z23.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z24.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z21.s, #0\n" + "mov z26.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z22.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "incw %[temp], all, mul #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z23.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z24.s, #0\n" + "ptrue p7.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "whilelt p2.s, %[temp], %[width]\n" "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "mov z25.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "mov z26.s, #0\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "mov z27.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z8.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z9.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "zip2 z11.b, z11.b, z12.b\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "udot z24.s, z8.b, z2.b[0]\n" @@ -2361,24 +2238,23 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" @@ -2386,12 +2262,13 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -2491,8 +2368,11 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip1 z14.b, z14.b, z12.b\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -2571,7 +2451,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2601,7 +2481,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -2631,7 +2511,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2660,31 +2540,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2706,33 +2586,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2754,33 +2634,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2802,33 +2682,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2850,34 +2730,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -2942,7 +2824,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2972,7 +2854,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3002,7 +2884,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3031,31 +2913,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3077,33 +2959,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3125,33 +3007,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3173,33 +3055,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3221,7 +3103,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -3240,7 +3122,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -3253,175 +3135,109 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "whilelt p6.b, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "whilelt p4.b, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" + "add a_ptr3, a_ptr2, %[lda]\n" "mov z19.s, #0\n" - "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "mov z20.s, #0\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "mov z21.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "mov z22.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "whilelt p6.b, %[temp], %[leftovers]\n" "mov z23.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "whilelt p0.s, %[temp], %[width]\n" "mov z24.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "whilelt p4.b, %[temp], %[width]\n" "mov z25.s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" - "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "incw %[temp], all, mul #1\n" "mov z26.s, #0\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "mov z27.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip2 z9.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "mov z28.s, #0\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "mov z29.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" + "zip2 z11.b, z8.b, z9.b\n" + "incw %[temp], all, mul #1\n" + "zip1 z9.b, z8.b, z9.b\n" + "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "mov z30.s, #0\n" - "zip2 z15.b, z12.b, z13.b\n" - "zip1 z13.b, z12.b, z13.b\n" + "ptrue p7.b\n" "mov z31.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "zip2 z12.b, z10.b, z8.b\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "incw %[temp], all, mul #1\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "zip1 z8.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip2 z9.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z11.b, z11.b, z12.b\n" "add a_ptr3, a_ptr3, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1rqb z6.b, p7/z, [a_ptr2]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z7.b, p7/z, [a_ptr3]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p7/z, [a_ptr2]\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z8.b, z14.b, z12.b\n" - "ld1rqb z7.b, p7/z, [a_ptr3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "add a_ptr3, a_ptr3, #0x20\n" "udot z26.s, z10.b, z2.b[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -3649,29 +3465,29 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z28.s, z8.b, z3.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "udot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p7/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3797,9 +3613,13 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr3, a_ptr3, #2\n" "zip1 z14.b, z14.b, z12.b\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -3893,7 +3713,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3927,7 +3747,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3961,7 +3781,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3994,31 +3814,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4044,33 +3864,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4096,33 +3916,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4148,33 +3968,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4200,37 +4020,40 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z28.s, z8.b, z3.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "udot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p6/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" @@ -4309,7 +4132,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4343,7 +4166,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -4377,7 +4200,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4410,31 +4233,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4460,33 +4283,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4512,33 +4335,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4564,33 +4387,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4616,7 +4439,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -4641,11 +4464,12 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp deleted file mode 100644 index 06622d6f2e..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - - - -namespace arm_gemm -{ - -// Actual kernel implementations -void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int); - -class smallK_fp32_mla_1VLx4 -{ -public: - typedef float operand_type; - typedef float result_type; - - typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int); - - /* Kernel blocking parameters */ - static unsigned int out_height() - { - return 4; - } - - static unsigned int out_width() - { - return get_vector_length() * 1; - } - - static unsigned int k_unroll() - { - return 1; - } - - - - // Default to the generic kernel - kern_type kernel=sve_smallK_fp32_mla_1VLx4; - - smallK_fp32_mla_1VLx4(const CPUInfo *ci) - { - - } -}; - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp deleted file mode 100644 index e2cc1d14e2..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp +++ /dev/null @@ -1,4264 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __ARM_FEATURE_SVE - -#include - - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -namespace arm_gemm { - -void sve_smallK_fp32_mla_1VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); - - const long loops_count = M / 4; - const long oddrow_count = M % 4; - const long ldab = lda * sizeof(float); - const long ldcb = ldc * sizeof(float); - const long odd_depth = K % 4; - const float *betaptr = β - long ldbb = ldb * sizeof(float); - - for (int x0=0; x0() * 1)) { - const long width = std::min((unsigned long)N-x0, (get_vector_length() * 1)); - long loops = loops_count; - long oddrows = oddrow_count; - long temp = 0; - const float *b_ptr0 = B + x0; - - const float *a_ptr0 = A; - - float *c_ptr0 = C + x0; - - switch(K) { - case 1: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 2: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 3: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 4: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 5: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 6: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 7: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 8: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 9: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 10: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 11: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 12: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 13: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 14: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 15: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 16: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 17: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 18: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 19: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 20: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 21: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 22: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 23: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z26.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - default: - case 24: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z26.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z27.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "fmla z28.s, z27.s, z0.s[3]\n" - "fmla z29.s, z27.s, z1.s[3]\n" - "fmla z30.s, z27.s, z2.s[3]\n" - "fmla z31.s, z27.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "fmla z28.s, z27.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - } - } -} - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp deleted file mode 100644 index 022efdfc26..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - - - -namespace arm_gemm -{ - -// Actual kernel implementations -void sve_smallK_hybrid_fp32_mla_1VLx4(const float *, int, const float *, float *, int, float, int, int, int); - -class smallK_hybrid_fp32_mla_1VLx4 -{ -public: - typedef float operand_type; - typedef float result_type; - - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); - - /* Kernel blocking parameters */ - static unsigned int out_height() - { - return 4; - } - - static unsigned int out_width() - { - return get_vector_length() * 1; - } - - static unsigned int k_unroll() - { - return 1; - } - - StdTransformsSVE transforms = {}; - - // Default to the generic kernel - kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx4; - - smallK_hybrid_fp32_mla_1VLx4(const CPUInfo *ci) - { - - } -}; - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp deleted file mode 100644 index 3e7e713106..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp +++ /dev/null @@ -1,4004 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __ARM_FEATURE_SVE - -#include - - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -namespace arm_gemm { - -void sve_smallK_hybrid_fp32_mla_1VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); - - const long loops_count = M / 4; - const long oddrow_count = M % 4; - const long ldab = lda * sizeof(float); - const long ldcb = ldc * sizeof(float); - const int K_stride = K; - const long odd_depth = K % 4; - const float *betaptr = β - - for (int x0=0; x0() * 1)) { - const long width = std::min((unsigned long)N-x0, (get_vector_length() * 1)); - long loops = loops_count; - long oddrows = oddrow_count; - long temp = 0; - const float *b_ptr0 = B + (K_stride * x0); - - const float *a_ptr0 = A; - - float *c_ptr0 = C + x0; - - switch(K) { - case 1: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 2: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 3: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 4: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 5: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 6: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 7: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 8: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 9: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 10: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 11: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 12: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 13: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 14: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 15: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 16: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 17: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 18: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 19: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 20: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 21: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 22: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 23: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - default: - case 24: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z27.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "fmla z28.s, z27.s, z0.s[3]\n" - "fmla z29.s, z27.s, z1.s[3]\n" - "fmla z30.s, z27.s, z2.s[3]\n" - "fmla z31.s, z27.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "fmla z28.s, z27.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - } - } -} - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp index ca90ee18ce..fc18cbdbbf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp @@ -31,7 +31,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_1VLx8 { @@ -39,7 +39,7 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -57,15 +57,27 @@ public: return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE transforms = {}; // Default to the generic kernel kern_type kernel=sve_smallK_hybrid_s8s32_dot_1VLx8; - smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) - { - - } + smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp index 04605b2ac1..d9813a5cdf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp @@ -25,13 +25,15 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { +void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)get_vector_length()) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp index 453301417d..51d3e736ed 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp @@ -31,7 +31,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_1VLx8 { @@ -39,7 +39,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -57,15 +57,27 @@ public: return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE transforms = {}; // Default to the generic kernel kern_type kernel=sve_smallK_hybrid_u8u32_dot_1VLx8; - smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) - { - - } + smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp index 7c965d38a6..6eed457152 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp @@ -25,13 +25,15 @@ #include +#include "arm_gemm.hpp" + #include #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { +void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)get_vector_length()) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); diff --git a/src/core/NEON/kernels/arm_gemm/mergeresults.cpp b/src/core/NEON/kernels/arm_gemm/mergeresults.cpp new file mode 100644 index 0000000000..83d6bccf2b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/mergeresults.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017-2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* As some of the merges need these headers, but are all included in the + * arm_gemm namespace, put these headers here. */ +#include + +#include + +#include "arm_gemm.hpp" +#include "asmlib.hpp" +#include "utils.hpp" + +namespace arm_gemm { + +template +void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append) { + // For SVE cases, multiply the width up by the vector length. + // Use the *input* type to determine this, since this will be what the kernel operated on. + const int width = twidth * (sve ? get_vector_length() : 1); + + const int full_y_blocks = (ymax - y0) / height; + const int y_remainder = (ymax - y0) % height; + const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0); + + const int full_x_blocks = (xmax - x0) / width; + const int x_remainder = (xmax - x0) % width; + const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0); + + for (int y_block = 0; y_block < y_blocks; y_block++) { + int ybase = y0 + (y_block * height); + + int fill_rows = (y_block < full_y_blocks) ? height : y_remainder; + + for (int x_block = 0; x_block < x_blocks; x_block++) { + int xbase = x0 + (x_block * width); + + int fill_cols = (x_block < full_x_blocks) ? width : x_remainder; + + for (int row=0; row < fill_rows; row++) { + for (int col=0; col < fill_cols; col++) { + Tout &r = out[(ybase + row) * ldc + xbase + col]; + Tout v = in[row * width + col]; + + if (append) { + v += r; + } + + if (bias) { + v += bias[xbase + col]; + } + + switch(act.type) { + default: + case Activation::Type::None: + break; + + case Activation::Type::ReLU: + v = std::max(v, static_cast(0)); + break; + + case Activation::Type::BoundedReLU: + v = std::max(std::min(v, static_cast(act.param1)), static_cast(0)); + break; + } + + r = v; + } + } + + in += (width * height); + } + } +} + +#include "merges/list.hpp" + +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) +template void MergeResults<12u, 8u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool); +#endif + +#if defined(__arm__) && defined(__ARM_FP16_ARGS) +template void MergeResults<8u, 6u, false, float, __fp16>(__fp16*, float const*, int, int, int, int, int, __fp16 const*, Activation, bool); +#endif + +} // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/mergeresults.hpp b/src/core/NEON/kernels/arm_gemm/mergeresults.hpp index 04d1343b1c..1aedd23a4b 100644 --- a/src/core/NEON/kernels/arm_gemm/mergeresults.hpp +++ b/src/core/NEON/kernels/arm_gemm/mergeresults.hpp @@ -23,58 +23,9 @@ */ #pragma once -/* As some of the merges need these headers, but are all included in the - * arm_gemm namespace, put these headers here. */ -#include - -#include "asmlib.hpp" -#include "utils.hpp" - namespace arm_gemm { template -inline void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout alpha, const Tout beta) { - // For SVE cases, multiply the width up by the vector length. - // Use the *input* type to determine this, since this will be what the kernel operated on. - const int width = twidth * (sve ? get_vector_length() : 1); - - const int full_y_blocks = (ymax - y0) / height; - const int y_remainder = (ymax - y0) % height; - const int y_blocks = full_y_blocks + (y_remainder ? 1 : 0); - - const int full_x_blocks = (xmax - x0) / width; - const int x_remainder = (xmax - x0) % width; - const int x_blocks = full_x_blocks + (x_remainder ? 1 : 0); - - for (int y_block = 0; y_block < y_blocks; y_block++) { - int ybase = y0 + (y_block * height); - - int fill_rows = (y_block < full_y_blocks) ? height : y_remainder; - - for (int x_block = 0; x_block < x_blocks; x_block++) { - int xbase = x0 + (x_block * width); - - int fill_cols = (x_block < full_x_blocks) ? width : x_remainder; - - for (int row=0; row < fill_rows; row++) { - for (int col=0; col < fill_cols; col++) { - Tout &p = out[(ybase + row) * ldc + xbase + col]; - - // Special case for beta==0 - don't read the input; - // (0 * x == 0) is not always true for FP types. - if (beta == static_cast(0)) { - p = (alpha * in[row * width + col]); - } else { - p = (p * beta) + (alpha * in[row * width + col]); - } - } - } - - in += (width * height); - } - } -} - -#include "merges/list.hpp" +void MergeResults(Tout * out, const Tin * in, int ldc, int y0, int ymax, int x0, int xmax, const Tout *bias, Activation act, bool append); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp b/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp index e1af2d4490..9409646818 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/a32_merge_float_8x6.hpp @@ -28,13 +28,35 @@ #include template<> -inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) { +void MergeResults<8, 6, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) { const float *inptr = in; prefetch_6x(inptr); prefetch_6x(inptr + 96); - float32x4_t av = vdupq_n_f32(alpha); - float32x4_t bv = vdupq_n_f32(beta); + float nullbias[8]; + float minval = - std::numeric_limits::infinity(); + float maxval = std::numeric_limits::infinity(); + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + float32x4_t minv = vdupq_n_f32(minval); + float32x4_t maxv = vdupq_n_f32(maxval); + + if (!append && !bias) + { + memset(nullbias, 0, (8 * sizeof(float))); + } for (int y=y0; y(float *out, const float *in, const int ldo switch ((y + 5) - ymax) { case 4: outptr1 = dummyres; - // fall through case 3: outptr2 = dummyres; - // fall through case 2: outptr3 = dummyres; - // fall through case 1: outptr4 = dummyres; - // fall through case 0: outptr5 = dummyres; break; @@ -80,24 +98,24 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo } } - if (beta == 0.0f) { - /* If beta=0, don't read the original input at all. */ + if (append) { + /* Append mode: Read, activate, write. */ /* For ragged X, manually copy over the valid results. */ if ((i+7) >= xmax) { for (int xi=0; xi<8; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 8]); + *outptr1 = std::min(std::max(minval, inptr[xi + 8] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 16]); + *outptr2 = std::min(std::max(minval, inptr[xi + 16] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 24]); + *outptr3 = std::min(std::max(minval, inptr[xi + 24] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 32]); + *outptr4 = std::min(std::max(minval, inptr[xi + 32] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 40]); + *outptr5 = std::min(std::max(minval, inptr[xi + 40] + *outptr5), maxval); outptr5++; } } @@ -107,69 +125,100 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo __asm __volatile ( // Rows 0-1 "VLD1.32 {d0-d3}, [%[inptr]]!\n" + "VLD1.32 {d8-d11}, [%[outptr0]]\n" "VLD1.32 {d4-d7}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[outptr1]]\n" - "VMUL.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[inptr], #352]") - "VMUL.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q2\n" + "VADD.f32 q7, q7, q3\n" ASM_PREFETCH("[%[inptr], #416]") - "VMUL.f32 q6, q2, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[inptr], #480]") - "VMUL.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr1]]!\n" // Rows 2-3 "VLD1.32 {d0-d3}, [%[inptr]]!\n" + "VLD1.32 {d8-d11}, [%[outptr2]]\n" "VLD1.32 {d4-d7}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[outptr3]]\n" - "VMUL.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr0], #96]") - "VMUL.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q2\n" + "VADD.f32 q7, q7, q3\n" ASM_PREFETCH("[%[outptr1], #96]") - "VMUL.f32 q6, q2, %q[av]\n" - ASM_PREFETCH("[%[outptr2], #96]") - "VMUL.f32 q7, q3, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" + ASM_PREFETCH("[%[outptr2], #128]") + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr3]]!\n" // Rows 4-5 "VLD1.32 {d0-d3}, [%[inptr]]!\n" + "VLD1.32 {d8-d11}, [%[outptr4]]\n" "VLD1.32 {d4-d7}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[outptr5]]\n" - "VMUL.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr3], #96]") - "VMUL.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr4]]!\n" - ASM_PREFETCH("[%[outptr4], #96]") - "VMUL.f32 q6, q2, %q[av]\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q2\n" + "VADD.f32 q7, q7, q3\n" + ASM_PREFETCH("[%[outptr4], #128]") + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[outptr5], #128]") - "VMUL.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr4]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr5]]!\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7" + : [minv] "w" (minv), [maxv] "w" (maxv) + : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "memory" ); } } else { - /* Non-zero beta: Read output and apply beta. */ + /* Bias mode: Add bias to everything, then min/max/write as before. */ + const float *biasptr = bias ? bias + i : nullbias; /* For ragged X, manually copy over the valid results. */ if ((i+7) >= xmax) { - for (int xi=0; xi<8; xi++) { + for (int xi=0; xi<7; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 8]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 8] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 16]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 16] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 24]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 32]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 32] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 40]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 40] + biasptr[xi]), maxval); outptr5++; } } @@ -178,68 +227,75 @@ inline void MergeResults<8, 6, false>(float *out, const float *in, const int ldo /* Optimized routine to copy an entire block */ __asm __volatile ( // Rows 0-1 - "VLD1.32 {d8-d11}, [%[outptr0]]\n" - "VMUL.f32 q4, q4, %q[bv]\n" - "VLD1.32 {d12-d15}, [%[outptr1]]\n" - "VMUL.f32 q5, q5, %q[bv]\n" - "VLD1.32 {d0-d3}, [%[inptr]]!\n" - "VMUL.f32 q6, q6, %q[bv]\n" - "VLD1.32 {d4-d7}, [%[inptr]]!\n" - "VMUL.f32 q7, q7, %q[bv]\n" + "VLD1.32 {d8-d11}, [%[inptr]]!\n" + "VLD1.32 {d0-d3}, [%[biasptr]]\n" + "VLD1.32 {d12-d15}, [%[inptr]]!\n" - "VMLA.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[inptr], #352]") - "VMLA.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q0\n" + "VADD.f32 q7, q7, q1\n" ASM_PREFETCH("[%[inptr], #416]") - "VMLA.f32 q6, q2, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[inptr], #480]") - "VMLA.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr0]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr1]]!\n" // Rows 2-3 - "VLD1.32 {d8-d11}, [%[outptr2]]\n" - "VMUL.f32 q4, q4, %q[bv]\n" - "VLD1.32 {d12-d15}, [%[outptr3]]\n" - "VMUL.f32 q5, q5, %q[bv]\n" - "VLD1.32 {d0-d3}, [%[inptr]]!\n" - "VMUL.f32 q6, q6, %q[bv]\n" - "VLD1.32 {d4-d7}, [%[inptr]]!\n" - "VMUL.f32 q7, q7, %q[bv]\n" + "VLD1.32 {d8-d11}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[inptr]]!\n" - "VMLA.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr0], #96]") - "VMLA.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q0\n" + "VADD.f32 q7, q7, q1\n" ASM_PREFETCH("[%[outptr1], #96]") - "VMLA.f32 q6, q2, %q[av]\n" - ASM_PREFETCH("[%[outptr2], #96]") - "VMLA.f32 q7, q3, %q[av]\n" + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" + ASM_PREFETCH("[%[outptr2], #128]") + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr2]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr3]]!\n" // Rows 4-5 - "VLD1.32 {d8-d11}, [%[outptr4]]\n" - "VMUL.f32 q4, q4, %q[bv]\n" - "VLD1.32 {d12-d15}, [%[outptr5]]\n" - "VMUL.f32 q5, q5, %q[bv]\n" - "VLD1.32 {d0-d3}, [%[inptr]]!\n" - "VMUL.f32 q6, q6, %q[bv]\n" - "VLD1.32 {d4-d7}, [%[inptr]]!\n" - "VMUL.f32 q7, q7, %q[bv]\n" + "VLD1.32 {d8-d11}, [%[inptr]]!\n" + "VLD1.32 {d12-d15}, [%[inptr]]!\n" - "VMLA.f32 q4, q0, %q[av]\n" + "VADD.f32 q4, q4, q0\n" ASM_PREFETCH("[%[outptr3], #96]") - "VMLA.f32 q5, q1, %q[av]\n" - "VST1.32 {d8-d11}, [%[outptr4]]!\n" - ASM_PREFETCH("[%[outptr4], #96]") - "VMLA.f32 q6, q2, %q[av]\n" + "VADD.f32 q5, q5, q1\n" + "VADD.f32 q6, q6, q0\n" + "VADD.f32 q7, q7, q1\n" + ASM_PREFETCH("[%[outptr4], #128]") + "VMAX.f32 q4, q4, %q[minv]\n" + "VMAX.f32 q5, q5, %q[minv]\n" + "VMAX.f32 q6, q6, %q[minv]\n" ASM_PREFETCH("[%[outptr5], #128]") - "VMLA.f32 q7, q3, %q[av]\n" + "VMAX.f32 q7, q7, %q[minv]\n" + "VMIN.f32 q4, q4, %q[maxv]\n" + "VMIN.f32 q5, q5, %q[maxv]\n" + "VST1.32 {d8-d11}, [%[outptr4]]!\n" + "VMIN.f32 q6, q6, %q[maxv]\n" + "VMIN.f32 q7, q7, %q[maxv]\n" "VST1.32 {d12-d15}, [%[outptr5]]!\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7" + : [minv] "w" (minv), [maxv] "w" (maxv), [biasptr] "r" (biasptr) + : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "memory" ); } } diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp deleted file mode 100644 index 9fca4e3a84..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_12x8.hpp +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -template<> -inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) { - const float *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 96); - - float32x4_t av = vdupq_n_f32(alpha); - float32x4_t bv = vdupq_n_f32(beta); - - for (int y=y0; y= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (beta==0.0f) { - /* If beta==0, don't read the original input at all. */ - - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDP q0, q1, [%[inptr]]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr0]], #32\n" - ASM_PREFETCH("[%[inptr], #768]") - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr0]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr1]], #32\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr1]], #16\n" - - // Rows 2-3 - "LDP q0, q1, [%[inptr], #96]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr2]], #32\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr2]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr3]], #32\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr3]], #16\n" - - // Rows 4-5 - "LDP q0, q1, [%[inptr], #192]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr4]], #32\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr4]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr5]], #32\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - "LDP q0, q1, [%[inptr], #288]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "STR q18, [%[outptr6]], #16\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } else { - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDP q16, q17, [%[outptr0]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr0], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr1]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr1], #32]\n" - ASM_PREFETCH("[%[inptr], #768]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr]]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr0]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr0]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr1]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr1]], #16\n" - - // Rows 2-3 - "LDP q16, q17, [%[outptr2]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr2], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr3]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr3], #32]\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #96]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr2]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr2]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr3]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr3]], #16\n" - - // Rows 4-5 - ASM_PREFETCH("[%[outptr0], #80]") - "LDP q16, q17, [%[outptr4]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr4], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr5]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr5], #32]\n" - ASM_PREFETCH("[%[outptr1], #80]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #192]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr2], #80]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr4]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr4]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr3], #80]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr5]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - ASM_PREFETCH("[%[outptr4], #80]") - "LDP q16, q17, [%[outptr6]]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDR q18, [%[outptr6], #32]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDP q19, q20, [%[outptr7]]\n" - "FMUL v18.4s, v18.4s, %[bv].4s\n" - "LDR q21, [%[outptr7], #32]\n" - ASM_PREFETCH("[%[outptr5], #80]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #288]\n" - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr6], #128]") - "FMLA v17.4s, v1.4s, %[av].4s\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q18, [%[outptr6]], #16\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr7], #128]") - "FMLA v20.4s, v4.4s, %[av].4s\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } - } - } -} - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp deleted file mode 100644 index 0e638eef1c..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_float_to_half_12x8.hpp +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -// This should be possible on any AArch64 target, but some old compilers don't support __fp16 arguments. -#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) - -#include - -template<> -inline void MergeResults<12,8,false>(__fp16 *out, const float *in, int ldout, int y0, int ymax, int x0, int xmax, const __fp16 alpha, const __fp16 beta) { - const float *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 24); - - float32x4_t av = vdupq_n_f32(alpha); - float32x4_t bv = vdupq_n_f32(beta); - - for (int y=y0; y= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (beta == ((__fp16)0.0f)) { - /* If beta==0, don't read the output. */ - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDP q0, q1, [%[inptr]]\n" - "LDP q2, q3, [%[inptr], #32]\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #768]") - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #832]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[inptr], #896]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr0]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr0]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr1]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr1]], #8\n" - - // Rows 2-3 - "LDP q0, q1, [%[inptr], #96]\n" - "LDP q2, q3, [%[inptr], #128]\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[outptr0], #64]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[outptr1], #64]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr2]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr2]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr3]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr3]], #8\n" - - // Rows 4-5 - "LDP q0, q1, [%[inptr], #192]\n" - "LDP q2, q3, [%[inptr], #224]\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr2], #64]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[outptr3], #64]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[outptr4], #88]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr4]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr4]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr5]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr5]], #8\n" - - // Rows 6-7 - "LDP q0, q1, [%[inptr], #288]\n" - "LDP q2, q3, [%[inptr], #320]\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMUL v16.4s, v0.4s, %[av].4s\n" - "FMUL v17.4s, v1.4s, %[av].4s\n" - ASM_PREFETCH("[%[outptr5], #64]") - "FCVTN v16.4h, v16.4s\n" - ASM_PREFETCH("[%[outptr6], #88]") - "FCVTN2 v16.8h, v17.4s\n" - ASM_PREFETCH("[%[outptr7], #88]") - "FMUL v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr6]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr6]], #8\n" - "FMUL v19.4s, v3.4s, %[av].4s\n" - "FMUL v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr7]], #16\n" - "FMUL v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr7]], #8\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } else { - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 96; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Rows 0-1 - "LDR q16, [%[outptr0]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr0], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr1]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr1], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr]]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[inptr], #768]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - ASM_PREFETCH("[%[inptr], #960]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr0]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr0]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr1]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr1]], #8\n" - - // Rows 2-3 - "LDR q16, [%[outptr2]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr2], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr3]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr3], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #96]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[inptr], #1024]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr0], #64]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr1], #64]") - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr2]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr2]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr3]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr3]], #8\n" - - // Rows 4-5 - "LDR q16, [%[outptr4]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr4], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr5]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr5], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #192]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[outptr2], #64]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[outptr3], #64]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr4], #88]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr4]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr4]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr5]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr5]], #8\n" - - // Rows 6-7 - "LDR q16, [%[outptr6]]\n" - "FCVTL2 v17.4s, v16.8h\n" - "LDR d18, [%[outptr6], #16]\n" - "FCVTL v16.4s, v16.4h\n" - "LDR q19, [%[outptr7]]\n" - "FMUL v17.4s, v17.4s, %[bv].4s\n" - "LDR d21, [%[outptr7], #16]\n" - "FMUL v16.4s, v16.4s, %[bv].4s\n" - "LDP q0, q1, [%[inptr], #288]\n" - "FCVTL v18.4s, v18.4h\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FCVTL2 v20.4s, v19.8h\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FCVTL v19.4s, v19.4h\n" - ASM_PREFETCH("[%[outptr5], #64]") - "FCVTL v21.4s, v21.4h\n" - ASM_PREFETCH("[%[outptr6], #88]") - "FMUL v18.4s, v18.4s, %[bv].4s\n" - ASM_PREFETCH("[%[outptr7], #88]") - "FMUL v20.4s, v20.4s, %[bv].4s\n" - "FMUL v19.4s, v19.4s, %[bv].4s\n" - "FMUL v21.4s, v21.4s, %[bv].4s\n" - "FMLA v16.4s, v0.4s, %[av].4s\n" - "FMLA v17.4s, v1.4s, %[av].4s\n" - "FCVTN v16.4h, v16.4s\n" - "FCVTN2 v16.8h, v17.4s\n" - "FMLA v18.4s, v2.4s, %[av].4s\n" - "STR q16, [%[outptr6]], #16\n" - "FCVTN v18.4h, v18.4s\n" - "STR d18, [%[outptr6]], #8\n" - "FMLA v19.4s, v3.4s, %[av].4s\n" - "FMLA v20.4s, v4.4s, %[av].4s\n" - "FCVTN v19.4h, v19.4s\n" - "FCVTN2 v19.8h, v20.4s\n" - "STR q19, [%[outptr7]], #16\n" - "FMLA v21.4s, v5.4s, %[av].4s\n" - "FCVTN v21.4h, v21.4s\n" - "STR d21, [%[outptr7]], #8\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [av] "w" (av), [bv] "w" (bv) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } - } - } -} - -#endif // __aarch64__ && __ARM_FP16_ARGS diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp new file mode 100644 index 0000000000..7bfab412ca --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp @@ -0,0 +1,2074 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) + +template<> +void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const __fp16 *bias, Activation act, bool append) +{ + const __fp16 *inptr = in; + __fp16 nullbias[24] = { 0 }; + __fp16 minval = - static_cast<__fp16>(std::numeric_limits::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits::infinity()); + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (24 * sizeof(__fp16))); + } + + for (int y=y0; y= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( +#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + ".arch armv8.2-a+fp16\n" +#endif + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr0]]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q13, [%[outptr1]]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q15, [%[outptr4], #0x10]\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval); + outptr5++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "str q17, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q10, [%[outptr5], #0x10]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add %[inptr], %[inptr], #0x180\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + *outptr6)), maxval); + outptr6++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "str q17, [%[outptr5]]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr6]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q13, [%[outptr6], #0x10]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q14, [%[outptr6], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "add %[inptr], %[inptr], #0x180\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + *outptr0)), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + *outptr1)), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + *outptr2)), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + *outptr3)), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + *outptr4)), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + *outptr5)), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + *outptr6)), maxval); + outptr6++; + *outptr7 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 168] + *outptr7)), maxval); + outptr7++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "str q10, [%[outptr0]]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr1]]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr2]]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr3]]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.8h, v10.8h, v2.8h\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr4]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.8h, v10.8h, v0.8h\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.8h, v11.8h, v3.8h\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.8h, v12.8h, v4.8h\n" + "ldr q7, [%[outptr7]]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.8h, v10.8h, v1.8h\n" + "ldr q15, [%[inptr], #0x150]\n" + "fmin v11.8h, v11.8h, v0.8h\n" + "ldr q8, [%[outptr7], #0x10]\n" + "fmin v12.8h, v12.8h, v0.8h\n" + "str q17, [%[outptr5]]\n" + "fadd v13.8h, v13.8h, v5.8h\n" + "ldr q16, [%[inptr], #0x160]\n" + "fadd v14.8h, v14.8h, v6.8h\n" + "ldr q9, [%[outptr7], #0x20]\n" + "fmax v11.8h, v11.8h, v1.8h\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v12.8h, v12.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x170]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v15.8h, v15.8h, v7.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q12, [%[outptr6]]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr6], #0x10]\n" + "fadd v16.8h, v16.8h, v8.8h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.8h, v17.8h, v9.8h\n" + "str q14, [%[outptr6], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q15, [%[outptr7]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q16, [%[outptr7], #0x10]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q17, [%[outptr7], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add %[outptr7], %[outptr7], #0x30\n" + "add %[inptr], %[inptr], #0x180\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + else + { + const __fp16 *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q13, [%[outptr0]]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr1]]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "str q19, [%[outptr2]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr1]]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr3]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "str q19, [%[outptr2]]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr3]]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr4]]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval); + outptr5++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "ldr q13, [%[inptr], #0x100]\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr3]]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x110]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fadd v20.8h, v20.8h, v2.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "str q17, [%[outptr4]]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fadd v13.8h, v13.8h, v3.8h\n" + "fadd v14.8h, v14.8h, v4.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr5]]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + biasptr[xi])), maxval); + outptr6++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr3]]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.8h, v20.8h, v2.8h\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.8h, v13.8h, v3.8h\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr4]]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.8h, v14.8h, v4.8h\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "fadd v15.8h, v15.8h, v2.8h\n" + "str q19, [%[outptr4], #0x20]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q20, [%[outptr5]]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "fadd v16.8h, v16.8h, v3.8h\n" + "fadd v17.8h, v17.8h, v4.8h\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr6]]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+23) >= xmax) + { + for (int xi=0; xi<23; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi] + biasptr[xi])), maxval); + outptr0++; + *outptr1 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 24] + biasptr[xi])), maxval); + outptr1++; + *outptr2 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 48] + biasptr[xi])), maxval); + outptr2++; + *outptr3 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 72] + biasptr[xi])), maxval); + outptr3++; + *outptr4 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 96] + biasptr[xi])), maxval); + outptr4++; + *outptr5 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 120] + biasptr[xi])), maxval); + outptr5++; + *outptr6 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 144] + biasptr[xi])), maxval); + outptr6++; + *outptr7 = std::min(std::max(minval, static_cast<__fp16>(inptr[xi + 168] + biasptr[xi])), maxval); + outptr7++; + } + } + inptr += 192; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "dup v0.8h, %[maxval].h[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.8h, %[minval].h[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd v13.8h, v13.8h, v2.8h\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.8h, v14.8h, v3.8h\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.8h, v15.8h, v4.8h\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.8h, v16.8h, v2.8h\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "str q13, [%[outptr0]]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.8h, v17.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.8h, v18.8h, v4.8h\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.8h, v19.8h, v2.8h\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "add %[outptr0], %[outptr0], #0x30\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "str q16, [%[outptr1]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.8h, v20.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.8h, v13.8h, v4.8h\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.8h, v14.8h, v2.8h\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "add %[outptr1], %[outptr1], #0x30\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q19, [%[outptr2]]\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.8h, v15.8h, v3.8h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.8h, v16.8h, v4.8h\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.8h, v17.8h, v2.8h\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "add %[outptr2], %[outptr2], #0x30\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "str q14, [%[outptr3]]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.8h, v18.8h, v3.8h\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fadd v19.8h, v19.8h, v4.8h\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.8h, v20.8h, v2.8h\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.8h, v13.8h, v3.8h\n" + "add %[outptr3], %[outptr3], #0x30\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "str q17, [%[outptr4]]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "fmin v13.8h, v13.8h, v0.8h\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.8h, v14.8h, v4.8h\n" + "ldr q18, [%[inptr], #0x150]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "fmax v13.8h, v13.8h, v1.8h\n" + "fmin v14.8h, v14.8h, v0.8h\n" + "str q19, [%[outptr4], #0x20]\n" + "fadd v15.8h, v15.8h, v2.8h\n" + "ldr q19, [%[inptr], #0x160]\n" + "fadd v16.8h, v16.8h, v3.8h\n" + "add %[outptr4], %[outptr4], #0x30\n" + "fmax v14.8h, v14.8h, v1.8h\n" + "str q20, [%[outptr5]]\n" + "fmin v15.8h, v15.8h, v0.8h\n" + "ldr q20, [%[inptr], #0x170]\n" + "fmin v16.8h, v16.8h, v0.8h\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v17.8h, v17.8h, v4.8h\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v15.8h, v15.8h, v1.8h\n" + "fmax v16.8h, v16.8h, v1.8h\n" + "fadd v18.8h, v18.8h, v2.8h\n" + "str q14, [%[outptr5], #0x20]\n" + "fmin v17.8h, v17.8h, v0.8h\n" + "add %[outptr5], %[outptr5], #0x30\n" + "fadd v19.8h, v19.8h, v3.8h\n" + "str q15, [%[outptr6]]\n" + "fmin v18.8h, v18.8h, v0.8h\n" + "fmax v17.8h, v17.8h, v1.8h\n" + "fadd v20.8h, v20.8h, v4.8h\n" + "str q16, [%[outptr6], #0x10]\n" + "fmin v19.8h, v19.8h, v0.8h\n" + "fmax v18.8h, v18.8h, v1.8h\n" + "fmin v20.8h, v20.8h, v0.8h\n" + "str q17, [%[outptr6], #0x20]\n" + "fmax v19.8h, v19.8h, v1.8h\n" + "add %[outptr6], %[outptr6], #0x30\n" + "fmax v20.8h, v20.8h, v1.8h\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ && (FP16_KERNELS || __ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp index fcdca59bdd..0c5aa7eae8 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp32_12x8.hpp @@ -26,11 +26,33 @@ #ifdef __aarch64__ template<> -inline void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) +void MergeResults<12, 8, false>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) { const float *inptr = in; + float nullbias[12] = { 0 }; + float minval = - std::numeric_limits::infinity(); + float maxval = std::numeric_limits::infinity(); - for (int y=y0; y(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (12 * sizeof(float))); + } + + for (int y=y0; y(float *out, const float *in, const int ld const int height = ymax - y; - for (int i=x0; i= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; } } @@ -62,23 +86,34 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "add %[inptr], %[inptr], #0x180\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr0]]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" "add %[outptr0], %[outptr0], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -88,13 +123,13 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; } } @@ -102,35 +137,55 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q7, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q13, [%[outptr1]]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "str q15, [%[outptr1], #0x20]\n" "add %[outptr1], %[outptr1], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -140,15 +195,15 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; } } @@ -156,47 +211,76 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr0], #0x10]\n" - "ldr q4, [%[inptr], #0x40]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q5, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q7, [%[inptr], #0x50]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q4, [%[inptr], #0x80]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v10.4s, v10.4s, v2.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" "add %[outptr2], %[outptr2], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -206,17 +290,17 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; } } @@ -224,58 +308,96 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q5, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q6, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q6, [%[inptr], #0x80]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v13.4s, v13.4s, v5.4s\n" "str q10, [%[outptr2], #0x20]\n" - "ldr q7, [%[inptr], #0xb0]\n" + "fmax v11.4s, v11.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "add %[outptr3], %[outptr3], #0x30\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr3]]\n" "add %[inptr], %[inptr], #0x180\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -285,19 +407,19 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; } } @@ -305,70 +427,117 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q6, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr1], #0x10]\n" - "ldr q7, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr2], #0x10]\n" - "ldr q4, [%[inptr], #0xa0]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr3], #0x10]\n" - "ldr q5, [%[inptr], #0xd0]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr4], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q7, [%[inptr], #0x50]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q4, [%[inptr], #0x80]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q5, [%[inptr], #0xb0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr3]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q6, [%[inptr], #0xe0]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" - "add %[outptr4], %[outptr4], #0x30\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q15, [%[outptr4], #0x10]\n" "add %[inptr], %[inptr], #0x180\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -378,21 +547,21 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval); outptr5++; } } @@ -400,82 +569,138 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q6, [%[inptr], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q7, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q4, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x10]\n" - "ldr q5, [%[inptr], #0xa0]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x10]\n" - "ldr q6, [%[inptr], #0xd0]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x10]\n" - "ldr q7, [%[inptr], #0x100]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.4s, v15.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q6, [%[inptr], #0x80]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.4s, v17.4s, v9.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q7, [%[inptr], #0xb0]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q4, [%[inptr], #0xe0]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "str q17, [%[outptr5]]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q5, [%[inptr], #0x110]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q10, [%[outptr5], #0x10]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q11, [%[outptr5], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -485,23 +710,23 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + *outptr6), maxval); outptr6++; } } @@ -509,94 +734,159 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q6, [%[inptr], #0x120]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q7, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr0], #0x10]\n" - "ldr q4, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q5, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q6, [%[inptr], #0xa0]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr3], #0x10]\n" - "ldr q7, [%[inptr], #0xd0]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr4], #0x10]\n" - "ldr q4, [%[inptr], #0x100]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr5], #0x10]\n" - "ldr q5, [%[inptr], #0x130]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr6], #0x10]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q7, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr3]]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.4s, v12.4s, v4.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q4, [%[inptr], #0x80]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.4s, v10.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q5, [%[inptr], #0xb0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "str q17, [%[outptr5]]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax v12.4s, v12.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q6, [%[inptr], #0xe0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" - "ldr q7, [%[inptr], #0x110]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr6]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q13, [%[outptr6], #0x10]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x20]\n" - "ldr q4, [%[inptr], #0x140]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr6], #0x20]\n" + "str q14, [%[outptr6], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "add %[outptr6], %[outptr6], #0x30\n" "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -607,25 +897,25 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); + *outptr0 = std::min(std::max(minval, inptr[xi] + *outptr0), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + *outptr1), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + *outptr2), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + *outptr3), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + *outptr4), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + *outptr5), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + *outptr6), maxval); outptr6++; - *outptr7 = (alpha * inptr[xi + 84]); + *outptr7 = std::min(std::max(minval, inptr[xi + 84] + *outptr7), maxval); outptr7++; } } @@ -633,105 +923,179 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q4, [%[inptr]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[outptr0]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q10, [%[inptr]]\n" + "ldr q3, [%[outptr0], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q5, [%[inptr], #0x30]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q6, [%[inptr], #0x60]\n" + "ldr q11, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q7, [%[inptr], #0x90]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr1]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x30]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "ldr q8, [%[outptr2]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "str q10, [%[outptr0]]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q16, [%[inptr], #0x60]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x70]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr3]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x90]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr1]]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0xa0]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0xb0]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr4]]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0xc0]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr4], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr2]]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0xd0]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0xe0]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr5]]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xf0]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q2, [%[outptr5], #0x10]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "str q11, [%[outptr3]]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q6, [%[inptr], #0x120]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q7, [%[inptr], #0x150]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7]]\n" - "ldr q4, [%[inptr], #0x10]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q5, [%[inptr], #0x40]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q6, [%[inptr], #0x70]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q4, [%[inptr], #0xd0]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x10]\n" - "ldr q5, [%[inptr], #0x100]\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x10]\n" - "ldr q6, [%[inptr], #0x130]\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x10]\n" - "ldr q7, [%[inptr], #0x160]\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x10]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q5, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v7.4s\n" + "ldr q10, [%[inptr], #0x100]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr3], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q11, [%[inptr], #0x110]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q4, [%[outptr6]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "ldr q12, [%[inptr], #0x120]\n" + "fadd v10.4s, v10.4s, v2.4s\n" + "ldr q5, [%[outptr6], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr4]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q13, [%[inptr], #0x130]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "fmin v10.4s, v10.4s, v0.4s\n" + "str q15, [%[outptr4], #0x10]\n" + "fadd v11.4s, v11.4s, v3.4s\n" + "ldr q14, [%[inptr], #0x140]\n" + "fadd v12.4s, v12.4s, v4.4s\n" + "ldr q7, [%[outptr7]]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "fmax v10.4s, v10.4s, v1.4s\n" + "ldr q15, [%[inptr], #0x150]\n" + "fmin v11.4s, v11.4s, v0.4s\n" + "ldr q8, [%[outptr7], #0x10]\n" + "fmin v12.4s, v12.4s, v0.4s\n" + "str q17, [%[outptr5]]\n" + "fadd v13.4s, v13.4s, v5.4s\n" + "ldr q16, [%[inptr], #0x160]\n" + "fadd v14.4s, v14.4s, v6.4s\n" + "ldr q9, [%[outptr7], #0x20]\n" + "fmax v11.4s, v11.4s, v1.4s\n" + "str q10, [%[outptr5], #0x10]\n" + "fmax v12.4s, v12.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x170]\n" + "fmin v13.4s, v13.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q6, [%[inptr], #0x80]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "fadd v15.4s, v15.4s, v7.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q7, [%[inptr], #0xb0]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q12, [%[outptr6]]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr6], #0x10]\n" + "fadd v16.4s, v16.4s, v8.4s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd v17.4s, v17.4s, v9.4s\n" + "str q14, [%[outptr6], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q4, [%[inptr], #0xe0]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q15, [%[outptr7]]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q5, [%[inptr], #0x110]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "str q16, [%[outptr7], #0x10]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" - "ldr q6, [%[inptr], #0x140]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "str q17, [%[outptr7], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x20]\n" - "ldr q7, [%[inptr], #0x170]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "add %[outptr6], %[outptr6], #0x30\n" - "fmul v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" "add %[outptr7], %[outptr7], #0x30\n" "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -742,16 +1106,23 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { - switch(height) { + const float *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; } } @@ -759,29 +1130,34 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x10]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "add %[outptr0], %[outptr0], #0x30\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" "add %[inptr], %[inptr], #0x180\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -791,13 +1167,13 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; } } @@ -805,47 +1181,52 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x10]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q11, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x40]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q13, [%[outptr0]]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr1]]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" "add %[outptr1], %[outptr1], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -855,15 +1236,15 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; } } @@ -871,65 +1252,70 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x10]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr0], #0x10]\n" - "ldr q8, [%[outptr1], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x40]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q9, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x70]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q11, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x50]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q8, [%[outptr2], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr1], #0x20]\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x80]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "str q19, [%[outptr2]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" "add %[outptr2], %[outptr2], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -939,17 +1325,17 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; } } @@ -957,82 +1343,87 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr0], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x10]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q9, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x40]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q10, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x70]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q11, [%[outptr3], #0x10]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q10, [%[outptr2], #0x20]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr1]]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x80]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q11, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr2], #0x20]\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xb0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr3]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" "add %[outptr3], %[outptr3], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1042,19 +1433,19 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; } } @@ -1062,100 +1453,105 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr0], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x10]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr0], #0x10]\n" - "ldr q10, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x40]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr1], #0x10]\n" - "ldr q11, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x70]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr2], #0x10]\n" - "ldr q8, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xa0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr3], #0x10]\n" - "ldr q9, [%[outptr4], #0x10]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xd0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr4], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q11, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x50]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q8, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x80]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q9, [%[outptr3], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "str q19, [%[outptr2]]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xb0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q10, [%[outptr4], #0x20]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr3]]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "fadd v18.4s, v18.4s, v3.4s\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "str q16, [%[outptr3], #0x20]\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xe0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr4]]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" "add %[outptr4], %[outptr4], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1165,21 +1561,21 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval); outptr5++; } } @@ -1187,118 +1583,123 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr5]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q10, [%[outptr0], #0x10]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x10]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x10]\n" - "ldr q11, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x40]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x10]\n" - "ldr q8, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x70]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x10]\n" - "ldr q9, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xa0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x10]\n" - "ldr q10, [%[outptr4], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xd0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x10]\n" - "ldr q11, [%[outptr5], #0x10]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x100]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q10, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x80]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q11, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "fadd v18.4s, v18.4s, v3.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xb0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q8, [%[outptr4], #0x20]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr3]]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x110]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fadd v20.4s, v20.4s, v2.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xe0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q9, [%[outptr5], #0x20]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "str q17, [%[outptr4]]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fadd v13.4s, v13.4s, v3.4s\n" + "fadd v14.4s, v14.4s, v4.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q19, [%[outptr4], #0x20]\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x110]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr5]]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1308,23 +1709,23 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + biasptr[xi]), maxval); outptr6++; } } @@ -1332,136 +1733,141 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr5]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q10, [%[outptr6]]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x120]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q11, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x10]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr0], #0x10]\n" - "ldr q8, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x40]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr1], #0x10]\n" - "ldr q9, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x70]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr2], #0x10]\n" - "ldr q10, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xa0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr3], #0x10]\n" - "ldr q11, [%[outptr4], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xd0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr4], #0x10]\n" - "ldr q8, [%[outptr5], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x100]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr5], #0x10]\n" - "ldr q9, [%[outptr6], #0x10]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x130]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr6], #0x10]\n" - "ldr q10, [%[outptr0], #0x20]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x20]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr0], #0x20]\n" - "ldr q11, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x50]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr1], #0x20]\n" - "ldr q8, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x80]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr2], #0x20]\n" - "ldr q9, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xb0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr3], #0x20]\n" - "ldr q10, [%[outptr4], #0x20]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr3]]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.4s, v18.4s, v3.4s\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.4s, v20.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.4s, v13.4s, v3.4s\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0xe0]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr4], #0x20]\n" - "ldr q11, [%[outptr5], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr4]]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.4s, v14.4s, v4.4s\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "fadd v15.4s, v15.4s, v2.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x110]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr5], #0x20]\n" - "ldr q8, [%[outptr6], #0x20]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q20, [%[outptr5]]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "fadd v16.4s, v16.4s, v3.4s\n" + "fadd v17.4s, v17.4s, v4.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "str q14, [%[outptr5], #0x20]\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x140]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr6], #0x20]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr6]]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" "add %[outptr6], %[outptr6], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } @@ -1472,25 +1878,25 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld { if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) + for (int xi=0; xi<11; xi++) { if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); + *outptr0 = std::min(std::max(minval, inptr[xi] + biasptr[xi]), maxval); outptr0++; - *outptr1 = (alpha * inptr[xi + 12]) + (*outptr1 * beta); + *outptr1 = std::min(std::max(minval, inptr[xi + 12] + biasptr[xi]), maxval); outptr1++; - *outptr2 = (alpha * inptr[xi + 24]) + (*outptr2 * beta); + *outptr2 = std::min(std::max(minval, inptr[xi + 24] + biasptr[xi]), maxval); outptr2++; - *outptr3 = (alpha * inptr[xi + 36]) + (*outptr3 * beta); + *outptr3 = std::min(std::max(minval, inptr[xi + 36] + biasptr[xi]), maxval); outptr3++; - *outptr4 = (alpha * inptr[xi + 48]) + (*outptr4 * beta); + *outptr4 = std::min(std::max(minval, inptr[xi + 48] + biasptr[xi]), maxval); outptr4++; - *outptr5 = (alpha * inptr[xi + 60]) + (*outptr5 * beta); + *outptr5 = std::min(std::max(minval, inptr[xi + 60] + biasptr[xi]), maxval); outptr5++; - *outptr6 = (alpha * inptr[xi + 72]) + (*outptr6 * beta); + *outptr6 = std::min(std::max(minval, inptr[xi + 72] + biasptr[xi]), maxval); outptr6++; - *outptr7 = (alpha * inptr[xi + 84]) + (*outptr7 * beta); + *outptr7 = std::min(std::max(minval, inptr[xi + 84] + biasptr[xi]), maxval); outptr7++; } } @@ -1498,153 +1904,158 @@ inline void MergeResults<12, 8, false>(float *out, const float *in, const int ld } else { /* Optimized routine to copy an entire block */ __asm __volatile ( - "ldr q8, [%[outptr0]]\n" + "dup v0.4s, %[maxval].s[0]\n" + "ldr q2, [%[biasptr]]\n" + "dup v1.4s, %[minval].s[0]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "ldr q4, [%[biasptr], #0x20]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr]]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0]]\n" - "ldr q9, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x30]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1]]\n" - "ldr q10, [%[outptr2]]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x60]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2]]\n" - "ldr q11, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x90]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3]]\n" - "ldr q8, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xc0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4]]\n" - "ldr q9, [%[outptr5]]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0xf0]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5]]\n" - "ldr q10, [%[outptr6]]\n" + "fadd v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd v14.4s, v14.4s, v3.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "fadd v15.4s, v15.4s, v4.4s\n" + "ldr q19, [%[inptr], #0x60]\n" + "fadd v16.4s, v16.4s, v2.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "fmin v14.4s, v14.4s, v0.4s\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x120]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6]]\n" - "ldr q11, [%[outptr7]]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x150]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7]]\n" - "ldr q8, [%[outptr0], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x10]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x10]\n" - "ldr q9, [%[outptr1], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x40]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x10]\n" - "ldr q10, [%[outptr2], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x70]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x10]\n" - "ldr q11, [%[outptr3], #0x10]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xa0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x10]\n" - "ldr q8, [%[outptr4], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xd0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x10]\n" - "ldr q9, [%[outptr5], #0x10]\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x100]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x10]\n" - "ldr q10, [%[outptr6], #0x10]\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x130]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x10]\n" - "ldr q11, [%[outptr7], #0x10]\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x160]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x10]\n" - "ldr q8, [%[outptr0], #0x20]\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0x20]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr0], #0x20]\n" - "ldr q9, [%[outptr1], #0x20]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "str q13, [%[outptr0]]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "fadd v17.4s, v17.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "ldr q14, [%[inptr], #0x90]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "fadd v19.4s, v19.4s, v2.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "fmax v17.4s, v17.4s, v1.4s\n" "add %[outptr0], %[outptr0], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x50]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr1], #0x20]\n" - "ldr q10, [%[outptr2], #0x20]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "str q16, [%[outptr1]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "fadd v20.4s, v20.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd v13.4s, v13.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "fadd v14.4s, v14.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "fmax v20.4s, v20.4s, v1.4s\n" "add %[outptr1], %[outptr1], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x80]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr2], #0x20]\n" - "ldr q11, [%[outptr3], #0x20]\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q19, [%[outptr2]]\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "fadd v15.4s, v15.4s, v3.4s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd v16.4s, v16.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "fadd v17.4s, v17.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "fmax v15.4s, v15.4s, v1.4s\n" "add %[outptr2], %[outptr2], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0xb0]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr3], #0x20]\n" - "ldr q8, [%[outptr4], #0x20]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "str q14, [%[outptr3]]\n" + "fmin v17.4s, v17.4s, v0.4s\n" + "ldr q14, [%[inptr], #0x110]\n" + "fadd v18.4s, v18.4s, v3.4s\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fadd v19.4s, v19.4s, v4.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "fadd v20.4s, v20.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "fadd v13.4s, v13.4s, v3.4s\n" "add %[outptr3], %[outptr3], #0x30\n" - "fmul v8.4s, v8.4s, %[beta].s[0]\n" - "ldr q4, [%[inptr], #0xe0]\n" - "fmla v8.4s, v4.4s, %[alpha].s[0]\n" - "str q8, [%[outptr4], #0x20]\n" - "ldr q9, [%[outptr5], #0x20]\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "str q17, [%[outptr4]]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "ldr q17, [%[inptr], #0x140]\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "fmin v13.4s, v13.4s, v0.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "fadd v14.4s, v14.4s, v4.4s\n" + "ldr q18, [%[inptr], #0x150]\n" + "fmax v19.4s, v19.4s, v1.4s\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "fmax v13.4s, v13.4s, v1.4s\n" + "fmin v14.4s, v14.4s, v0.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "fadd v15.4s, v15.4s, v2.4s\n" + "ldr q19, [%[inptr], #0x160]\n" + "fadd v16.4s, v16.4s, v3.4s\n" "add %[outptr4], %[outptr4], #0x30\n" - "fmul v9.4s, v9.4s, %[beta].s[0]\n" - "ldr q5, [%[inptr], #0x110]\n" - "fmla v9.4s, v5.4s, %[alpha].s[0]\n" - "str q9, [%[outptr5], #0x20]\n" - "ldr q10, [%[outptr6], #0x20]\n" + "fmax v14.4s, v14.4s, v1.4s\n" + "str q20, [%[outptr5]]\n" + "fmin v15.4s, v15.4s, v0.4s\n" + "ldr q20, [%[inptr], #0x170]\n" + "fmin v16.4s, v16.4s, v0.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "fadd v17.4s, v17.4s, v4.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "fmax v15.4s, v15.4s, v1.4s\n" + "fmax v16.4s, v16.4s, v1.4s\n" + "fadd v18.4s, v18.4s, v2.4s\n" + "str q14, [%[outptr5], #0x20]\n" + "fmin v17.4s, v17.4s, v0.4s\n" "add %[outptr5], %[outptr5], #0x30\n" - "fmul v10.4s, v10.4s, %[beta].s[0]\n" - "ldr q6, [%[inptr], #0x140]\n" - "fmla v10.4s, v6.4s, %[alpha].s[0]\n" - "str q10, [%[outptr6], #0x20]\n" - "ldr q11, [%[outptr7], #0x20]\n" + "fadd v19.4s, v19.4s, v3.4s\n" + "str q15, [%[outptr6]]\n" + "fmin v18.4s, v18.4s, v0.4s\n" + "fmax v17.4s, v17.4s, v1.4s\n" + "fadd v20.4s, v20.4s, v4.4s\n" + "str q16, [%[outptr6], #0x10]\n" + "fmin v19.4s, v19.4s, v0.4s\n" + "fmax v18.4s, v18.4s, v1.4s\n" + "fmin v20.4s, v20.4s, v0.4s\n" + "str q17, [%[outptr6], #0x20]\n" + "fmax v19.4s, v19.4s, v1.4s\n" "add %[outptr6], %[outptr6], #0x30\n" - "fmul v11.4s, v11.4s, %[beta].s[0]\n" - "ldr q7, [%[inptr], #0x170]\n" - "fmla v11.4s, v7.4s, %[alpha].s[0]\n" - "str q11, [%[outptr7], #0x20]\n" + "fmax v20.4s, v20.4s, v1.4s\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" "add %[outptr7], %[outptr7], #0x30\n" - "add %[inptr], %[inptr], #0x180\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr) - : [alpha] "w" (alpha), [beta] "w" (beta) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory" + : [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" ); } } diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp deleted file mode 100644 index 60cc2f32da..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_half_24x8.hpp +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -// AArch64 only, and either the FP16_KERNELS option set or the target explicitly supports FP16 vectors. -#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) - -template<> -inline void MergeResults<24, 8>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, - const int x0, const int xmax, const __fp16 alpha, const __fp16 beta) { - const __fp16 *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 48); - - float16x8_t va = vdupq_n_f16(alpha); - float16x8_t vb = vdupq_n_f16(beta); - - for (int y=y0; y= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - - } - } - - if (beta == (__fp16)0.0f) { - /* If beta===0, don't read the output. */ - - /* For ragged X, manually copy over the valid results. */ - if ((i+23) >= xmax) { - for (int xi=0; xi<24; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]); - outptr0++; - *outptr1 = (alpha * inptr[xi + 24]); - outptr1++; - *outptr2 = (alpha * inptr[xi + 48]); - outptr2++; - *outptr3 = (alpha * inptr[xi + 72]); - outptr3++; - *outptr4 = (alpha * inptr[xi + 96]); - outptr4++; - *outptr5 = (alpha * inptr[xi + 120]); - outptr5++; - *outptr6 = (alpha * inptr[xi + 144]); - outptr6++; - *outptr7 = (alpha * inptr[xi + 168]); - outptr7++; - } - } - inptr += 192; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( -#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - ".arch armv8.2-a+fp16\n" -#endif - // Rows 0-1 - ASM_PREFETCH("[%[inptr], #768]") - "LDP q0, q1, [%[inptr]]\n" - "LDP q2, q3, [%[inptr], #32]\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr0]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr0]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr1]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr1]], #16\n" - ASM_PREFETCH("[%[inptr], #960]") - - // Rows 2-3 - ASM_PREFETCH("[%[inptr], #1024]") - "LDP q0, q1, [%[inptr], #96]\n" - "LDP q2, q3, [%[inptr], #128]\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr2]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr2]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr0], #80]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr3]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr3]], #16\n" - ASM_PREFETCH("[%[outptr1], #80]") - - // Rows 4-5 - ASM_PREFETCH("[%[outptr2], #80]") - "LDP q0, q1, [%[inptr], #192]\n" - "LDP q2, q3, [%[inptr], #224]\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr3], #80]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr4]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr4]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr4], #80]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr5]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - ASM_PREFETCH("[%[outptr5], #80]") - "LDP q0, q1, [%[inptr], #288]\n" - "LDP q2, q3, [%[inptr], #320]\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMUL v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr6], #128]") - "FMUL v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMUL v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr6]], #16\n" - "FMUL v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr7], #128]") - "FMUL v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMUL v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [va] "w" (va), [vb] "w" (vb) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } else { - /* For ragged X, manually copy over the valid results. */ - if ((i+23) >= xmax) { - for (int xi=0; xi<24; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (alpha * inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (alpha * inptr[xi + 24]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (alpha * inptr[xi + 48]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (alpha * inptr[xi + 72]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (alpha * inptr[xi + 96]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (alpha * inptr[xi + 120]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (alpha * inptr[xi + 144]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (alpha * inptr[xi + 168]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 192; - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( -#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - ".arch armv8.2-a+fp16\n" -#endif - // Rows 0-1 - "LDP q16, q17, [%[outptr0]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr0], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr1]]\n" - "FMUL v18.8h, v18.8h, %[vb].8h\n" - ASM_PREFETCH("[%[inptr], #768]") - "LDR q21, [%[outptr1], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr]]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #32]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #64]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #832]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr0]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr0]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #896]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr1]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr1]], #16\n" - ASM_PREFETCH("[%[inptr], #960]") - - // Rows 2-3 - "LDP q16, q17, [%[outptr2]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr2], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr3]]\n" - "FMUL v18.8h, v18.8h, %[vb].8h\n" - ASM_PREFETCH("[%[inptr], #1024]") - "LDR q21, [%[outptr3], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr], #96]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #128]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #160]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[inptr], #1088]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr2]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr2]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr0], #80]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr3]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr3]], #16\n" - ASM_PREFETCH("[%[outptr1], #80]") - - // Rows 4-5 - "LDP q16, q17, [%[outptr4]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr4], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr5]]\n" - "FMUL v18.8h, v18.8h, %[vb].8h\n" - ASM_PREFETCH("[%[outptr2], #80]") - "LDR q21, [%[outptr5], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr], #192]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #224]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #256]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr3], #80]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr4]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr4]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr4], #80]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr5]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr5]], #16\n" - - // Rows 6-7 - "LDP q16, q17, [%[outptr6]]\n" - "FMUL v16.8h, v16.8h, %[vb].8h\n" - "LDR q18, [%[outptr6], #32]\n" - "FMUL v17.8h, v17.8h, %[vb].8h\n" - "LDP q19, q20, [%[outptr7]]\n" - ASM_PREFETCH("[%[outptr5], #80]") - "FMUL v18.8h, v18.8h, %[vb].8h\n" - "LDR q21, [%[outptr7], #32]\n" - "FMUL v19.8h, v19.8h, %[vb].8h\n" - "LDP q0, q1, [%[inptr], #288]\n" - "FMUL v20.8h, v20.8h, %[vb].8h\n" - "LDP q2, q3, [%[inptr], #320]\n" - "FMUL v21.8h, v21.8h, %[vb].8h\n" - "LDP q4, q5, [%[inptr], #352]\n" - "FMLA v16.8h, v0.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr6], #128]") - "FMLA v17.8h, v1.8h, %[va].8h\n" - "STP q16, q17, [%[outptr6]], #32\n" - "FMLA v18.8h, v2.8h, %[va].8h\n" - "STR q18, [%[outptr6]], #16\n" - "FMLA v19.8h, v3.8h, %[va].8h\n" - ASM_PREFETCH("[%[outptr7], #128]") - "FMLA v20.8h, v4.8h, %[va].8h\n" - "STP q19, q20, [%[outptr7]], #32\n" - "FMLA v21.8h, v5.8h, %[va].8h\n" - "STR q21, [%[outptr7]], #16\n" - "ADD %[inptr], %[inptr], #384\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : [va] "w" (va), [vb] "w" (vb) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21" - ); - } - } - } - } -} - -#endif // __aarch64__ && (FP16_KERNELS || __ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp deleted file mode 100644 index 410a0a1dc9..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_int32_12x8.hpp +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright (c) 2017-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -template<> -inline void MergeResults<12, 8, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t alpha, const int32_t beta) { - UNUSED(alpha); - const int32_t *inptr = in; - prefetch_6x(inptr); - prefetch_6x(inptr + 96); - - for (int y=y0; y= ymax) { - switch ((y + 7) - ymax) { - case 6: - outptr1 = dummyres; - // fall through - case 5: - outptr2 = dummyres; - // fall through - case 4: - outptr3 = dummyres; - // fall through - case 3: - outptr4 = dummyres; - // fall through - case 2: - outptr5 = dummyres; - // fall through - case 1: - outptr6 = dummyres; - // fall through - case 0: - outptr7 = dummyres; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - /* For ragged X, manually copy over the valid results. */ - if ((i+11) >= xmax) { - for (int xi=0; xi<12; xi++) { - if ((i+xi) < xmax) { - *outptr0 = (inptr[xi]) + (*outptr0 * beta); - outptr0++; - *outptr1 = (inptr[xi + 12]) + (*outptr1 * beta); - outptr1++; - *outptr2 = (inptr[xi + 24]) + (*outptr2 * beta); - outptr2++; - *outptr3 = (inptr[xi + 36]) + (*outptr3 * beta); - outptr3++; - *outptr4 = (inptr[xi + 48]) + (*outptr4 * beta); - outptr4++; - *outptr5 = (inptr[xi + 60]) + (*outptr5 * beta); - outptr5++; - *outptr6 = (inptr[xi + 72]) + (*outptr6 * beta); - outptr6++; - *outptr7 = (inptr[xi + 84]) + (*outptr7 * beta); - outptr7++; - } - } - inptr += 96; - } else { - if (beta == 0u) { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Row 0 - ASM_PREFETCH("[%x[outptr1], #192]") - "ldr q0, [%x[inptr]]\n" - "ldr q1, [%x[inptr], #0x10]\n" - "ldr q2, [%x[inptr], #0x20]\n" - - // Row 1 - ASM_PREFETCH("[%x[outptr2], #192]") - "ldr q3, [%x[inptr], #0x30]\n" - "str q0, [%x[outptr0]], #0x10\n" - "ldr q4, [%x[inptr], #0x40]\n" - "str q1, [%x[outptr0]], #0x10\n" - "ldr q5, [%x[inptr], #0x50]\n" - "str q2, [%x[outptr0]], #0x10\n" - - // Row 2 - ASM_PREFETCH("[%x[outptr3], #192]") - "ldr q0, [%x[inptr], #0x60]\n" - "str q3, [%x[outptr1]], #0x10\n" - "ldr q1, [%x[inptr], #0x70]\n" - "str q4, [%x[outptr1]], #0x10\n" - "ldr q2, [%x[inptr], #0x80]\n" - "str q5, [%x[outptr1]], #0x10\n" - - // Row 3 - ASM_PREFETCH("[%x[outptr4], #192]") - "ldr q3, [%x[inptr], #0x90]\n" - "str q0, [%x[outptr2]], #0x10\n" - "ldr q4, [%x[inptr], #0xa0]\n" - "str q1, [%x[outptr2]], #0x10\n" - "ldr q5, [%x[inptr], #0xb0]\n" - "str q2, [%x[outptr2]], #0x10\n" - - // Row 4 - ASM_PREFETCH("[%x[outptr5], #192]") - "ldr q0, [%x[inptr], #0xc0]\n" - "str q3, [%x[outptr3]], #0x10\n" - "ldr q1, [%x[inptr], #0xd0]\n" - "str q4, [%x[outptr3]], #0x10\n" - "ldr q2, [%x[inptr], #0xe0]\n" - "str q5, [%x[outptr3]], #0x10\n" - - // Row 5 - ASM_PREFETCH("[%x[outptr6], #192]") - "ldr q3, [%x[inptr], #0xf0]\n" - "str q0, [%x[outptr4]], #0x10\n" - "ldr q4, [%x[inptr], #0x100]\n" - "str q1, [%x[outptr4]], #0x10\n" - "ldr q5, [%x[inptr], #0x110]\n" - "str q2, [%x[outptr4]], #0x10\n" - - // Row 6 - ASM_PREFETCH("[%x[outptr7], #192]") - "ldr q0, [%x[inptr], #0x120]\n" - "str q3, [%x[outptr5]], #0x10\n" - "ldr q1, [%x[inptr], #0x130]\n" - "str q4, [%x[outptr5]], #0x10\n" - "ldr q2, [%x[inptr], #0x140]\n" - "str q5, [%x[outptr5]], #0x10\n" - - // Row 7 - "ldr q3, [%x[inptr], #0x150]\n" - "str q0, [%x[outptr6]], #0x10\n" - "ldr q4, [%x[inptr], #0x160]\n" - "str q1, [%x[outptr6]], #0x10\n" - "ldr q5, [%x[inptr], #0x170]\n" - "str q2, [%x[outptr6]], #0x10\n" - "str q3, [%x[outptr7]], #0x10\n" - "str q4, [%x[outptr7]], #0x10\n" - "str q5, [%x[outptr7]], #0x10\n" - - "add %x[inptr], %x[inptr], #0x180\n" - : [outptr0] "+r" (outptr0), - [outptr1] "+r" (outptr1), - [outptr2] "+r" (outptr2), - [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), - [outptr5] "+r" (outptr5), - [outptr6] "+r" (outptr6), - [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6" - ); - } else { - /* Optimized routine to copy an entire block */ - __asm __volatile ( - // Row 0 - ASM_PREFETCH("[%x[outptr1], #192]") - "ldr q3, [%x[outptr0]]\n" - "ldr q4, [%x[outptr0], #0x10]\n" - "ldr q5, [%x[outptr0], #0x20]\n" - "ldr q6, [%x[inptr]]\n" - "ldr q7, [%x[inptr], #0x10]\n" - "ldr q8, [%x[inptr], #0x20]\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr1]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr1], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr1], #0x20]\n" - - // Row 1 - ASM_PREFETCH("[%x[outptr2], #192]") - "ldr q6, [%x[inptr], #0x30]\n" - "str q3, [%x[outptr0]], #0x10\n" - "ldr q7, [%x[inptr], #0x40]\n" - "str q4, [%x[outptr0]], #0x10\n" - "ldr q8, [%x[inptr], #0x50]\n" - "str q5, [%x[outptr0]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "ldr q3, [%x[outptr2]]\n" - "add v1.4s, v1.4s, v7.4s\n" - "ldr q4, [%x[outptr2], #0x10]\n" - "add v2.4s, v2.4s, v8.4s\n" - "ldr q5, [%x[outptr2], #0x20]\n" - - // Row 2 - ASM_PREFETCH("[%x[outptr3], #192]") - "ldr q6, [%x[inptr], #0x60]\n" - "str q0, [%x[outptr1]], #0x10\n" - "ldr q7, [%x[inptr], #0x70]\n" - "str q1, [%x[outptr1]], #0x10\n" - "ldr q8, [%x[inptr], #0x80]\n" - "str q2, [%x[outptr1]], #0x10\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr3]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr3], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr3], #0x20]\n" - - // Row 3 - ASM_PREFETCH("[%x[outptr4], #192]") - "ldr q6, [%x[inptr], #0x90]\n" - "str q3, [%x[outptr2]], #0x10\n" - "ldr q7, [%x[inptr], #0xa0]\n" - "str q4, [%x[outptr2]], #0x10\n" - "ldr q8, [%x[inptr], #0xb0]\n" - "str q5, [%x[outptr2]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "ldr q3, [%x[outptr4]]\n" - "add v1.4s, v1.4s, v7.4s\n" - "ldr q4, [%x[outptr4], #0x10]\n" - "add v2.4s, v2.4s, v8.4s\n" - "ldr q5, [%x[outptr4], #0x20]\n" - - // Row 4 - ASM_PREFETCH("[%x[outptr5], #192]") - "ldr q6, [%x[inptr], #0xc0]\n" - "str q0, [%x[outptr3]], #0x10\n" - "ldr q7, [%x[inptr], #0xd0]\n" - "str q1, [%x[outptr3]], #0x10\n" - "ldr q8, [%x[inptr], #0xe0]\n" - "str q2, [%x[outptr3]], #0x10\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr5]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr5], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr5], #0x20]\n" - - // Row 5 - ASM_PREFETCH("[%x[outptr6], #192]") - "ldr q6, [%x[inptr], #0xf0]\n" - "str q3, [%x[outptr4]], #0x10\n" - "ldr q7, [%x[inptr], #0x100]\n" - "str q4, [%x[outptr4]], #0x10\n" - "ldr q8, [%x[inptr], #0x110]\n" - "str q5, [%x[outptr4]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "ldr q3, [%x[outptr6]]\n" - "add v1.4s, v1.4s, v7.4s\n" - "ldr q4, [%x[outptr6], #0x10]\n" - "add v2.4s, v2.4s, v8.4s\n" - "ldr q5, [%x[outptr6], #0x20]\n" - - // Row 6 - ASM_PREFETCH("[%x[outptr7], #192]") - "ldr q6, [%x[inptr], #0x120]\n" - "str q0, [%x[outptr5]], #0x10\n" - "ldr q7, [%x[inptr], #0x130]\n" - "str q1, [%x[outptr5]], #0x10\n" - "ldr q8, [%x[inptr], #0x140]\n" - "str q2, [%x[outptr5]], #0x10\n" - "add v3.4s, v3.4s, v6.4s\n" - "ldr q0, [%x[outptr7]]\n" - "add v4.4s, v4.4s, v7.4s\n" - "ldr q1, [%x[outptr7], #0x10]\n" - "add v5.4s, v5.4s, v8.4s\n" - "ldr q2, [%x[outptr7], #0x20]\n" - - // Row 7 - "ldr q6, [%x[inptr], #0x150]\n" - "str q3, [%x[outptr6]], #0x10\n" - "ldr q7, [%x[inptr], #0x160]\n" - "str q4, [%x[outptr6]], #0x10\n" - "ldr q8, [%x[inptr], #0x170]\n" - "str q5, [%x[outptr6]], #0x10\n" - "add v0.4s, v0.4s, v6.4s\n" - "add v1.4s, v1.4s, v7.4s\n" - "add v2.4s, v2.4s, v8.4s\n" - "str q0, [%x[outptr7]], #0x10\n" - "str q1, [%x[outptr7]], #0x10\n" - "str q2, [%x[outptr7]], #0x10\n" - - "add %x[inptr], %x[inptr], #0x180\n" - : [outptr0] "+r" (outptr0), - [outptr1] "+r" (outptr1), - [outptr2] "+r" (outptr2), - [outptr3] "+r" (outptr3), - [outptr4] "+r" (outptr4), - [outptr5] "+r" (outptr5), - [outptr6] "+r" (outptr6), - [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8" - ); - - } - } - } - } -} - -template<> -inline void MergeResults<12, 8>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t alpha, const uint32_t beta) { - // Since the above code uses only MUL and MLA instructions discard the "unsignedness" and proceed safely. - MergeResults<12, 8>(reinterpret_cast(out), reinterpret_cast(in), ldout, y0, ymax, x0, xmax, static_cast(alpha), static_cast(beta)); -} - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp new file mode 100644 index 0000000000..313f8295ff --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_12x8.hpp @@ -0,0 +1,1595 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<12, 8, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const int32_t *inptr = in; + int32_t nullbias[12] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (12 * sizeof(int32_t))); + } + + for (int y=y0; y= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "ldr q11, [%[inptr], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "add %[inptr], %[inptr], #0x180\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q13, [%[outptr1]]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "str q17, [%[outptr2], #0x10]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q14, [%[inptr], #0x40]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q3, [%[outptr3]]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr3]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "str q15, [%[outptr4], #0x10]\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q16, [%[outptr2]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "str q10, [%[outptr5], #0x10]\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "str q16, [%[outptr2]]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "str q13, [%[outptr6], #0x10]\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + *outptr7 += inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "ldr q7, [%[outptr7]]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "ldr q15, [%[inptr], #0x150]\n" + "ldr q8, [%[outptr7], #0x10]\n" + "ldr q16, [%[inptr], #0x160]\n" + "str q13, [%[outptr6], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr7], #0x20]\n" + "ldr q17, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr7]]\n" + "str q16, [%[outptr7], #0x10]\n" + "str q17, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + else + { + const int32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "ldr q13, [%[inptr]]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v2.4s\n" + "add v14.4s, v14.4s, v3.4s\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x30]\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v14.4s, v14.4s, v3.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "add v16.4s, v16.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "str q14, [%[outptr0], #0x10]\n" + "add v17.4s, v17.4s, v3.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "add v19.4s, v19.4s, v2.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "add v14.4s, v14.4s, v2.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v3.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "str q17, [%[outptr1], #0x10]\n" + "ldr q17, [%[inptr], #0xc0]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + *outptr7 = biasptr[xi] + inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "add v18.4s, v18.4s, v3.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "str q20, [%[outptr2], #0x10]\n" + "ldr q20, [%[inptr], #0xf0]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "ldr q18, [%[inptr], #0x150]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v18.4s, v18.4s, v2.4s\n" + "ldr q19, [%[inptr], #0x160]\n" + "str q20, [%[outptr5]]\n" + "ldr q20, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v19.4s, v19.4s, v3.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "add v20.4s, v20.4s, v4.4s\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp new file mode 100644 index 0000000000..a93060fa5a --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_s32_4x4.hpp @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<4, 4, false>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const int32_t *inptr = in; + int32_t nullbias[4] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (4 * sizeof(int32_t))); + } + + for (int y=y0; y= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + *outptr3 += inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr3]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "add %[inptr], %[inptr], #0x40\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q13, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + else + { + const int32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v13.4s, v13.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "ldr q14, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add v13.4s, v13.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v14.4s, v14.4s, v2.4s\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "prfm PSTL1KEEP, [%[outptr3], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "str q14, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp new file mode 100644 index 0000000000..5569f5157b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_12x8.hpp @@ -0,0 +1,1595 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<12, 8, false>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const uint32_t *inptr = in; + uint32_t nullbias[12] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (12 * sizeof(uint32_t))); + } + + for (int y=y0; y= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "ldr q11, [%[inptr], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "add %[inptr], %[inptr], #0x180\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "ldr q14, [%[inptr], #0x40]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q7, [%[outptr1], #0x20]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q13, [%[outptr1]]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "str q17, [%[outptr2], #0x10]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "str q12, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "ldr q14, [%[inptr], #0x40]\n" + "ldr q7, [%[outptr1], #0x20]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q3, [%[outptr3]]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "str q12, [%[outptr3], #0x10]\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "ldr q8, [%[outptr2]]\n" + "ldr q16, [%[inptr], #0x60]\n" + "str q14, [%[outptr1], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "ldr q10, [%[inptr], #0x80]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr3]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "str q15, [%[outptr4], #0x10]\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "ldr q17, [%[inptr], #0x70]\n" + "str q15, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "ldr q2, [%[outptr2], #0x20]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q16, [%[outptr2]]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q10, [%[outptr2], #0x20]\n" + "ldr q6, [%[outptr4]]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "str q10, [%[outptr5], #0x10]\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "str q16, [%[outptr2]]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "ldr q4, [%[outptr3], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q17, [%[outptr2], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "str q13, [%[outptr6], #0x10]\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 12]; + outptr1++; + *outptr2 += inptr[xi + 24]; + outptr2++; + *outptr3 += inptr[xi + 36]; + outptr3++; + *outptr4 += inptr[xi + 48]; + outptr4++; + *outptr5 += inptr[xi + 60]; + outptr5++; + *outptr6 += inptr[xi + 72]; + outptr6++; + *outptr7 += inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ldr q3, [%[outptr0], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr0], #0x10]\n" + "ldr q13, [%[inptr], #0x30]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "ldr q6, [%[outptr1], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q12, [%[outptr0], #0x20]\n" + "ldr q14, [%[inptr], #0x40]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "ldr q7, [%[outptr1], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q13, [%[outptr1]]\n" + "ldr q15, [%[inptr], #0x50]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ldr q8, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q14, [%[outptr1], #0x10]\n" + "ldr q16, [%[inptr], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ldr q9, [%[outptr2], #0x10]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q15, [%[outptr1], #0x20]\n" + "ldr q17, [%[inptr], #0x70]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "ldr q2, [%[outptr2], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q16, [%[outptr2]]\n" + "ldr q10, [%[inptr], #0x80]\n" + "ldr q3, [%[outptr3]]\n" + "ldr q11, [%[inptr], #0x90]\n" + "str q17, [%[outptr2], #0x10]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q4, [%[outptr3], #0x10]\n" + "ldr q12, [%[inptr], #0xa0]\n" + "add v11.4s, v11.4s, v3.4s\n" + "ldr q5, [%[outptr3], #0x20]\n" + "ldr q13, [%[inptr], #0xb0]\n" + "str q10, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr4]]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0xc0]\n" + "ldr q7, [%[outptr4], #0x10]\n" + "ldr q15, [%[inptr], #0xd0]\n" + "str q12, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v6.4s\n" + "ldr q8, [%[outptr4], #0x20]\n" + "ldr q16, [%[inptr], #0xe0]\n" + "add v15.4s, v15.4s, v7.4s\n" + "str q13, [%[outptr3], #0x20]\n" + "ldr q9, [%[outptr5]]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0xf0]\n" + "ldr q2, [%[outptr5], #0x10]\n" + "ldr q10, [%[inptr], #0x100]\n" + "str q15, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v9.4s\n" + "ldr q3, [%[outptr5], #0x20]\n" + "ldr q11, [%[inptr], #0x110]\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q16, [%[outptr4], #0x20]\n" + "ldr q4, [%[outptr6]]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q17, [%[outptr5]]\n" + "ldr q12, [%[inptr], #0x120]\n" + "ldr q5, [%[outptr6], #0x10]\n" + "ldr q13, [%[inptr], #0x130]\n" + "str q10, [%[outptr5], #0x10]\n" + "add v12.4s, v12.4s, v4.4s\n" + "ldr q6, [%[outptr6], #0x20]\n" + "ldr q14, [%[inptr], #0x140]\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q11, [%[outptr5], #0x20]\n" + "ldr q7, [%[outptr7]]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "add v14.4s, v14.4s, v6.4s\n" + "str q12, [%[outptr6]]\n" + "ldr q15, [%[inptr], #0x150]\n" + "ldr q8, [%[outptr7], #0x10]\n" + "ldr q16, [%[inptr], #0x160]\n" + "str q13, [%[outptr6], #0x10]\n" + "add v15.4s, v15.4s, v7.4s\n" + "ldr q9, [%[outptr7], #0x20]\n" + "ldr q17, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v8.4s\n" + "str q14, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "add v17.4s, v17.4s, v9.4s\n" + "str q15, [%[outptr7]]\n" + "str q16, [%[outptr7], #0x10]\n" + "str q17, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + else + { + const uint32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "ldr q13, [%[inptr]]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v2.4s\n" + "add v14.4s, v14.4s, v3.4s\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "str q14, [%[outptr0], #0x10]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 2: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "ldr q15, [%[inptr], #0x20]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x30]\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v14.4s, v14.4s, v3.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "add v15.4s, v15.4s, v4.4s\n" + "str q13, [%[outptr0]]\n" + "add v16.4s, v16.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 3: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "str q14, [%[outptr0], #0x10]\n" + "add v17.4s, v17.4s, v3.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "add v19.4s, v19.4s, v2.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "str q17, [%[outptr1], #0x10]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 4: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "str q15, [%[outptr0], #0x20]\n" + "add %[outptr0], %[outptr0], #0x30\n" + "add v20.4s, v20.4s, v3.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v13.4s, v13.4s, v4.4s\n" + "str q16, [%[outptr1]]\n" + "add v14.4s, v14.4s, v2.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v15.4s, v15.4s, v3.4s\n" + "str q17, [%[outptr1], #0x10]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "str q19, [%[outptr2]]\n" + "str q20, [%[outptr2], #0x10]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 5: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "str q17, [%[outptr1], #0x10]\n" + "ldr q17, [%[inptr], #0xc0]\n" + "add v16.4s, v16.4s, v4.4s\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "str q14, [%[outptr3]]\n" + "str q15, [%[outptr3], #0x10]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 6: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "str q17, [%[outptr4]]\n" + "str q18, [%[outptr4], #0x10]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + case 7: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "ldr q19, [%[inptr], #0xe0]\n" + "add v18.4s, v18.4s, v3.4s\n" + "str q20, [%[outptr2], #0x10]\n" + "add v19.4s, v19.4s, v4.4s\n" + "ldr q20, [%[inptr], #0xf0]\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "str q20, [%[outptr5]]\n" + "str q13, [%[outptr5], #0x10]\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + default: + case 8: + { + if ((i+11) >= xmax) + { + for (int xi=0; xi<11; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 12]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 24]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 36]; + outptr3++; + *outptr4 = biasptr[xi] + inptr[xi + 48]; + outptr4++; + *outptr5 = biasptr[xi] + inptr[xi + 60]; + outptr5++; + *outptr6 = biasptr[xi] + inptr[xi + 72]; + outptr6++; + *outptr7 = biasptr[xi] + inptr[xi + 84]; + outptr7++; + } + } + inptr += 96; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "ldr q3, [%[biasptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ldr q4, [%[biasptr], #0x20]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ldr q13, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "ldr q14, [%[inptr], #0x10]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "add v13.4s, v13.4s, v2.4s\n" + "ldr q15, [%[inptr], #0x20]\n" + "ldr q16, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add v14.4s, v14.4s, v3.4s\n" + "str q13, [%[outptr0]]\n" + "add v15.4s, v15.4s, v4.4s\n" + "ldr q17, [%[inptr], #0x40]\n" + "add v16.4s, v16.4s, v2.4s\n" + "ldr q18, [%[inptr], #0x50]\n" + "ldr q19, [%[inptr], #0x60]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add v17.4s, v17.4s, v3.4s\n" + "str q14, [%[outptr0], #0x10]\n" + "add v18.4s, v18.4s, v4.4s\n" + "ldr q20, [%[inptr], #0x70]\n" + "add v19.4s, v19.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x80]\n" + "ldr q14, [%[inptr], #0x90]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add v20.4s, v20.4s, v3.4s\n" + "str q15, [%[outptr0], #0x20]\n" + "add v13.4s, v13.4s, v4.4s\n" + "ldr q15, [%[inptr], #0xa0]\n" + "add v14.4s, v14.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x30\n" + "str q16, [%[outptr1]]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "add v15.4s, v15.4s, v3.4s\n" + "ldr q16, [%[inptr], #0xb0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "str q17, [%[outptr1], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "add v16.4s, v16.4s, v4.4s\n" + "ldr q17, [%[inptr], #0xc0]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "str q18, [%[outptr1], #0x20]\n" + "add %[outptr1], %[outptr1], #0x30\n" + "add v17.4s, v17.4s, v2.4s\n" + "ldr q18, [%[inptr], #0xd0]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "str q19, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "add v18.4s, v18.4s, v3.4s\n" + "ldr q19, [%[inptr], #0xe0]\n" + "str q20, [%[outptr2], #0x10]\n" + "ldr q20, [%[inptr], #0xf0]\n" + "add v19.4s, v19.4s, v4.4s\n" + "str q13, [%[outptr2], #0x20]\n" + "add %[outptr2], %[outptr2], #0x30\n" + "add v20.4s, v20.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x100]\n" + "str q14, [%[outptr3]]\n" + "ldr q14, [%[inptr], #0x110]\n" + "add v13.4s, v13.4s, v3.4s\n" + "str q15, [%[outptr3], #0x10]\n" + "add v14.4s, v14.4s, v4.4s\n" + "ldr q15, [%[inptr], #0x120]\n" + "str q16, [%[outptr3], #0x20]\n" + "add %[outptr3], %[outptr3], #0x30\n" + "add v15.4s, v15.4s, v2.4s\n" + "ldr q16, [%[inptr], #0x130]\n" + "str q17, [%[outptr4]]\n" + "ldr q17, [%[inptr], #0x140]\n" + "add v16.4s, v16.4s, v3.4s\n" + "str q18, [%[outptr4], #0x10]\n" + "add v17.4s, v17.4s, v4.4s\n" + "ldr q18, [%[inptr], #0x150]\n" + "str q19, [%[outptr4], #0x20]\n" + "add %[outptr4], %[outptr4], #0x30\n" + "add v18.4s, v18.4s, v2.4s\n" + "ldr q19, [%[inptr], #0x160]\n" + "str q20, [%[outptr5]]\n" + "ldr q20, [%[inptr], #0x170]\n" + "add %[inptr], %[inptr], #0x180\n" + "add v19.4s, v19.4s, v3.4s\n" + "str q13, [%[outptr5], #0x10]\n" + "add v20.4s, v20.4s, v4.4s\n" + "str q14, [%[outptr5], #0x20]\n" + "add %[outptr5], %[outptr5], #0x30\n" + "str q15, [%[outptr6]]\n" + "str q16, [%[outptr6], #0x10]\n" + "str q17, [%[outptr6], #0x20]\n" + "add %[outptr6], %[outptr6], #0x30\n" + "str q18, [%[outptr7]]\n" + "str q19, [%[outptr7], #0x10]\n" + "str q20, [%[outptr7], #0x20]\n" + "add %[outptr7], %[outptr7], #0x30\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp new file mode 100644 index 0000000000..fd01bb2392 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_u32_4x4.hpp @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +template<> +void MergeResults<4, 4, false>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append) +{ + UNUSED(act); + + const uint32_t *inptr = in; + uint32_t nullbias[4] = { 0 }; + + + if (!append && !bias) + { + memset(nullbias, 0, (4 * sizeof(uint32_t))); + } + + for (int y=y0; y= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v10.4s, v10.4s, v2.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 += inptr[xi]; + outptr0++; + *outptr1 += inptr[xi + 4]; + outptr1++; + *outptr2 += inptr[xi + 8]; + outptr2++; + *outptr3 += inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[outptr0]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q10, [%[inptr]]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x20]\n" + "ldr q3, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x20]\n" + "add v10.4s, v10.4s, v2.4s\n" + "ldr q11, [%[inptr], #0x10]\n" + "ldr q4, [%[outptr2]]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x20]\n" + "ldr q12, [%[inptr], #0x20]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x20]\n" + "add v11.4s, v11.4s, v3.4s\n" + "str q10, [%[outptr0]]\n" + "ldr q5, [%[outptr3]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v12.4s, v12.4s, v4.4s\n" + "str q11, [%[outptr1]]\n" + "ldr q13, [%[inptr], #0x30]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "add %[inptr], %[inptr], #0x40\n" + "str q12, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "add v13.4s, v13.4s, v5.4s\n" + "str q13, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + else + { + const uint32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "add v11.4s, v11.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 2: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + case 3: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "add %[inptr], %[inptr], #0x40\n" + "add v13.4s, v13.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add %[outptr0], %[outptr0], #0x10\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + default: + case 4: + { + if ((i+3) >= xmax) + { + for (int xi=0; xi<3; xi++) + { + if ((i+xi) < xmax) + { + *outptr0 = biasptr[xi] + inptr[xi]; + outptr0++; + *outptr1 = biasptr[xi] + inptr[xi + 4]; + outptr1++; + *outptr2 = biasptr[xi] + inptr[xi + 8]; + outptr2++; + *outptr3 = biasptr[xi] + inptr[xi + 12]; + outptr3++; + } + } + inptr += 16; + } else { + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "ldr q2, [%[biasptr]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x40]\n" + "ldr q11, [%[inptr]]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x20]\n" + "ldr q12, [%[inptr], #0x10]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x20]\n" + "add v11.4s, v11.4s, v2.4s\n" + "ldr q13, [%[inptr], #0x20]\n" + "ldr q14, [%[inptr], #0x30]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x20]\n" + "add v12.4s, v12.4s, v2.4s\n" + "str q11, [%[outptr0]]\n" + "add v13.4s, v13.4s, v2.4s\n" + "add %[outptr0], %[outptr0], #0x10\n" + "add v14.4s, v14.4s, v2.4s\n" + "str q12, [%[outptr1]]\n" + "add %[outptr1], %[outptr1], #0x10\n" + "prfm PSTL1KEEP, [%[outptr3], #0x20]\n" + "add %[inptr], %[inptr], #0x40\n" + "str q13, [%[outptr2]]\n" + "add %[outptr2], %[outptr2], #0x10\n" + "str q14, [%[outptr3]]\n" + "add %[outptr3], %[outptr3], #0x10\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), + [inptr] "+r" (inptr) + : [biasptr] "r" (biasptr) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "memory" + ); + } + } + break; + + + } + } + } + } +} + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/merges/list.hpp b/src/core/NEON/kernels/arm_gemm/merges/list.hpp index 788a957d5e..4edb497967 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/list.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/list.hpp @@ -22,10 +22,13 @@ * SOFTWARE. */ #include "a32_merge_float_8x6.hpp" -#include "a64_merge_float_12x8.hpp" -#include "a64_merge_float_to_half_12x8.hpp" -#include "a64_merge_half_24x8.hpp" -#include "a64_merge_int32_12x8.hpp" +#include "a64_merge_fp16_24x8.hpp" +#include "a64_merge_fp32_12x8.hpp" +#include "a64_merge_s32_12x8.hpp" +#include "a64_merge_s32_4x4.hpp" +#include "a64_merge_u32_12x8.hpp" +#include "a64_merge_u32_4x4.hpp" +#include "sve_merge_fp16_3VLx8.hpp" #include "sve_merge_fp32_3VLx8.hpp" #include "sve_merge_s32_3VLx8.hpp" #include "sve_merge_u32_3VLx8.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp new file mode 100644 index 0000000000..e5efc09b52 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp16_3VLx8.hpp @@ -0,0 +1,1879 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __ARM_FEATURE_SVE + +template<> +void MergeResults<3, 8, true>(__fp16 *out, const __fp16 *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const __fp16 *bias, Activation act, bool append) +{ + const __fp16 *inptr = in; + __fp16 nullbias[384] = { 0 }; + __fp16 minval = - static_cast<__fp16>(std::numeric_limits::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits::infinity()); + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + + for (int y=y0; y())) + { + if (append) + { + switch(height) + { + case 1: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "fadd z12.h, z12.h, z4.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 2: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "fadd z14.h, z14.h, z6.h\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "addvl %[outptr1], %[outptr1], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 3: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z16.h, z16.h, z8.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fadd z17.h, z17.h, z9.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "addvl %[outptr2], %[outptr2], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 4: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z12.h, z12.h, z4.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "addvl %[outptr3], %[outptr3], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 5: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z14.h, z14.h, z6.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z15.h, z15.h, z7.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z16.h, z16.h, z8.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 6: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "ld1h z9.h, p0/z, [%[outptr5]]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z10.h, p1/z, [x8]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fadd z17.h, z17.h, z9.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "fmin z10.h, p1/m, z10.h, z0.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z11.h, p2/m, z11.h, z0.h\n" + "st1h z17.h, p0, [%[outptr5]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z10.h, p1/m, z10.h, z1.h\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z11.h, p2/m, z11.h, z1.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 7: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "ld1h z9.h, p0/z, [%[outptr5]]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z10.h, p1/z, [x8]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z4.h, p0/z, [%[outptr6]]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "ld1h z5.h, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "ld1h z13.h, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z6.h, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "fmin z10.h, p1/m, z10.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z10.h, p1/m, z10.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z11.h, p2/m, z11.h, z0.h\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd z13.h, z13.h, z5.h\n" + "st1h z17.h, p0, [%[outptr5]]\n" + "fmax z11.h, p2/m, z11.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z12.h, p0/m, z12.h, z0.h\n" + "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n" + "fmax z12.h, p0/m, z12.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z12.h, p0, [%[outptr6]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "addvl %[outptr4], %[outptr4], #3\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z13.h, p1, [%[outptr6], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "addvl %[outptr5], %[outptr5], #3\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "st1h z14.h, p2, [%[outptr6], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "addvl %[outptr6], %[outptr6], #3\n" + "addvl %[inptr], %[inptr], #24\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + default: + case 8: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[outptr0]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z10.h, p0/z, [%[inptr]]\n" + "inch %[p], all, mul #1\n" + "ld1h z5.h, p0/z, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "fadd z10.h, z10.h, z2.h\n" + "ld1h z3.h, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1h z11.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z13.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.h, p0/m, z10.h, z0.h\n" + "ld1h z4.h, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z12.h, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z6.h, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1h z14.h, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.h, p0/m, z10.h, z1.h\n" + "ld1h z7.h, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.h, p1/m, z11.h, z0.h\n" + "ld1h z15.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z8.h, p0/z, [%[outptr2]]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "st1h z10.h, p0, [%[outptr0]]\n" + "fadd z14.h, z14.h, z6.h\n" + "ld1h z16.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.h, p1/m, z11.h, z1.h\n" + "ld1h z9.h, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.h, p2/m, z12.h, z0.h\n" + "ld1h z17.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "ld1h z2.h, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "st1h z11.h, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z10.h, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.h, p2/m, z12.h, z1.h\n" + "ld1h z3.h, p0/z, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "ld1h z4.h, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z12.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z5.h, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p0, [%[outptr1]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z13.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "ld1h z6.h, p0/z, [%[outptr4]]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "st1h z14.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.h, p2/m, z10.h, z0.h\n" + "ld1h z14.h, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z15.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.h, p2/m, z10.h, z1.h\n" + "ld1h z8.h, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.h, p0/m, z11.h, z0.h\n" + "st1h z16.h, p0, [%[outptr2]]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.h, p1/m, z12.h, z0.h\n" + "ld1h z9.h, p0/z, [%[outptr5]]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.h, p0/m, z11.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "ld1h z2.h, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.h, p1/m, z12.h, z1.h\n" + "st1h z10.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z10.h, p1/z, [x8]\n" + "fadd z15.h, z15.h, z7.h\n" + "ld1h z3.h, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z11.h, p0, [%[outptr3]]\n" + "fadd z16.h, z16.h, z8.h\n" + "ld1h z11.h, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z4.h, p0/z, [%[outptr6]]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z12.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "ld1h z12.h, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "ld1h z5.h, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.h, z10.h, z2.h\n" + "st1h z13.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "ld1h z13.h, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z6.h, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z14.h, p0, [%[outptr4]]\n" + "fmin z10.h, p1/m, z10.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.h, z11.h, z3.h\n" + "ld1h z7.h, p0/z, [%[outptr7]]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z15.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z12.h, z12.h, z4.h\n" + "ld1h z15.h, p0/z, [x8, #5, MUL VL]\n" + "fmax z10.h, p1/m, z10.h, z1.h\n" + "ld1h z8.h, p1/z, [%[outptr7], #1, MUL VL]\n" + "fmin z11.h, p2/m, z11.h, z0.h\n" + "st1h z16.h, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z13.h, z13.h, z5.h\n" + "ld1h z16.h, p1/z, [x8, #6, MUL VL]\n" + "fmin z12.h, p0/m, z12.h, z0.h\n" + "ld1h z9.h, p2/z, [%[outptr7], #2, MUL VL]\n" + "fadd z14.h, z14.h, z6.h\n" + "st1h z17.h, p0, [%[outptr5]]\n" + "fmax z11.h, p2/m, z11.h, z1.h\n" + "ld1h z17.h, p2/z, [x8, #7, MUL VL]\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z12.h, p0/m, z12.h, z1.h\n" + "st1h z10.h, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "st1h z11.h, p2, [%[outptr5], #2, MUL VL]\n" + "fadd z15.h, z15.h, z7.h\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z12.h, p0, [%[outptr6]]\n" + "fadd z16.h, z16.h, z8.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z15.h, p0/m, z15.h, z0.h\n" + "st1h z13.h, p1, [%[outptr6], #1, MUL VL]\n" + "fadd z17.h, z17.h, z9.h\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.h, p1/m, z16.h, z0.h\n" + "st1h z14.h, p2, [%[outptr6], #2, MUL VL]\n" + "fmax z15.h, p0/m, z15.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z17.h, p2/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z16.h, p1/m, z16.h, z1.h\n" + "st1h z15.h, p0, [%[outptr7]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z17.h, p2/m, z17.h, z1.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "st1h z16.h, p1, [%[outptr7], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "addvl %[outptr5], %[outptr5], #3\n" + "st1h z17.h, p2, [%[outptr7], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "addvl %[outptr6], %[outptr6], #3\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "addvl %[outptr7], %[outptr7], #3\n" + "addvl %[inptr], %[inptr], #24\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + + } + } + else + { + const __fp16 *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { + case 1: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.h, %[p], %[w]\n" + "fadd z13.h, z13.h, z2.h\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "fadd z15.h, z15.h, z4.h\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 2: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "fadd z15.h, z15.h, z4.h\n" + "fadd z16.h, z16.h, z2.h\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fadd z17.h, z17.h, z3.h\n" + "fadd z18.h, z18.h, z4.h\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "addvl %[outptr1], %[outptr1], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 3: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "fadd z20.h, z20.h, z3.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.h, z13.h, z4.h\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "addvl %[outptr2], %[outptr2], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 4: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.h, z13.h, z4.h\n" + "fadd z14.h, z14.h, z2.h\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "fadd z15.h, z15.h, z3.h\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "addvl %[outptr3], %[outptr3], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 5: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z16.h, z16.h, z4.h\n" + "fadd z17.h, z17.h, z2.h\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "fadd z18.h, z18.h, z3.h\n" + "fadd z19.h, z19.h, z4.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 6: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.h, z17.h, z2.h\n" + "ld1h z13.h, p1/z, [x8]\n" + "fadd z18.h, z18.h, z3.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z19.h, z19.h, z4.h\n" + "fadd z20.h, z20.h, z2.h\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmin z20.h, p0/m, z20.h, z0.h\n" + "fadd z13.h, z13.h, z3.h\n" + "fadd z14.h, z14.h, z4.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "fmax z20.h, p0/m, z20.h, z1.h\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "st1h z20.h, p0, [%[outptr5]]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n" + "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + case 7: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.h, z17.h, z2.h\n" + "ld1h z13.h, p1/z, [x8]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.h, z18.h, z3.h\n" + "fadd z19.h, z19.h, z4.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "ld1h z15.h, p0/z, [x8, #2, MUL VL]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.h, z20.h, z2.h\n" + "ld1h z16.h, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.h, z13.h, z3.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "ld1h z17.h, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.h, p0/m, z20.h, z0.h\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.h, z14.h, z4.h\n" + "fadd z15.h, z15.h, z2.h\n" + "fmax z20.h, p0/m, z20.h, z1.h\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z20.h, p0, [%[outptr5]]\n" + "fmin z15.h, p0/m, z15.h, z0.h\n" + "fadd z16.h, z16.h, z3.h\n" + "fadd z17.h, z17.h, z4.h\n" + "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "fmax z15.h, p0/m, z15.h, z1.h\n" + "fmin z16.h, p1/m, z16.h, z0.h\n" + "fmin z17.h, p2/m, z17.h, z0.h\n" + "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" + "fmax z16.h, p1/m, z16.h, z1.h\n" + "st1h z15.h, p0, [%[outptr6]]\n" + "fmax z17.h, p2/m, z17.h, z1.h\n" + "st1h z16.h, p1, [%[outptr6], #1, MUL VL]\n" + "st1h z17.h, p2, [%[outptr6], #2, MUL VL]\n" + "addvl %[outptr6], %[outptr6], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + default: + case 8: + { + long w = xmax - i; + long p = 0; + /* Optimized routine to copy an entire block */ + __asm __volatile ( + "mov z0.h, %h[maxval]\n" + "addvl x8, %[inptr], #16\n" + "mov z1.h, %h[minval]\n" + "whilelt p0.h, %[p], %[w]\n" + "inch %[p], all, mul #1\n" + "prfm PLDL1KEEP, [%[inptr], #0x180]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1h z2.h, p0/z, [%[biasptr]]\n" + "whilelt p1.h, %[p], %[w]\n" + "ld1h z3.h, p0/z, [%[biasptr], #1, MUL VL]\n" + "inch %[p], all, mul #1\n" + "ld1h z4.h, p0/z, [%[biasptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" + "ld1h z13.h, p0/z, [%[inptr]]\n" + "whilelt p2.h, %[p], %[w]\n" + "ld1h z14.h, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.h, z13.h, z2.h\n" + "ld1h z15.h, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1h z16.h, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.h, z14.h, z3.h\n" + "ld1h z17.h, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.h, p0/m, z13.h, z0.h\n" + "ld1h z18.h, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.h, z15.h, z4.h\n" + "ld1h z19.h, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.h, z16.h, z2.h\n" + "ld1h z20.h, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.h, p1/m, z14.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.h, p0/m, z13.h, z1.h\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.h, p1/m, z14.h, z1.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.h, p2/m, z15.h, z0.h\n" + "st1h z13.h, p0, [%[outptr0]]\n" + "fmin z16.h, p0/m, z16.h, z0.h\n" + "ld1h z13.h, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.h, z17.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.h, p2/m, z15.h, z1.h\n" + "st1h z14.h, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.h, p0/m, z16.h, z1.h\n" + "ld1h z14.h, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.h, p1/m, z17.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fadd z18.h, z18.h, z4.h\n" + "st1h z15.h, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.h, z19.h, z2.h\n" + "ld1h z15.h, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.h, p1/m, z17.h, z1.h\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmin z18.h, p2/m, z18.h, z0.h\n" + "st1h z16.h, p0, [%[outptr1]]\n" + "fmin z19.h, p0/m, z19.h, z0.h\n" + "ld1h z16.h, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.h, z20.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.h, p2/m, z18.h, z1.h\n" + "st1h z17.h, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.h, p0/m, z19.h, z1.h\n" + "ld1h z17.h, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.h, p1/m, z20.h, z0.h\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.h, z13.h, z4.h\n" + "st1h z18.h, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.h, z14.h, z2.h\n" + "ld1h z18.h, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.h, p1/m, z20.h, z1.h\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmin z13.h, p2/m, z13.h, z0.h\n" + "st1h z19.h, p0, [%[outptr2]]\n" + "fmin z14.h, p0/m, z14.h, z0.h\n" + "ld1h z19.h, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.h, z15.h, z3.h\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.h, p2/m, z13.h, z1.h\n" + "st1h z20.h, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.h, p0/m, z14.h, z1.h\n" + "ld1h z20.h, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.h, p1/m, z15.h, z0.h\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fadd z16.h, z16.h, z4.h\n" + "st1h z13.h, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.h, z17.h, z2.h\n" + "ld1h z13.h, p1/z, [x8]\n" + "fmax z15.h, p1/m, z15.h, z1.h\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmin z16.h, p2/m, z16.h, z0.h\n" + "st1h z14.h, p0, [%[outptr3]]\n" + "fmin z17.h, p0/m, z17.h, z0.h\n" + "ld1h z14.h, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.h, z18.h, z3.h\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z16.h, p2/m, z16.h, z1.h\n" + "st1h z15.h, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z17.h, p0/m, z17.h, z1.h\n" + "ld1h z15.h, p0/z, [x8, #2, MUL VL]\n" + "fmin z18.h, p1/m, z18.h, z0.h\n" + "fadd z19.h, z19.h, z4.h\n" + "st1h z16.h, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.h, z20.h, z2.h\n" + "ld1h z16.h, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.h, z13.h, z3.h\n" + "addvl %[outptr3], %[outptr3], #3\n" + "fmax z18.h, p1/m, z18.h, z1.h\n" + "st1h z17.h, p0, [%[outptr4]]\n" + "fmin z19.h, p2/m, z19.h, z0.h\n" + "ld1h z17.h, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.h, p0/m, z20.h, z0.h\n" + "fmin z13.h, p1/m, z13.h, z0.h\n" + "st1h z18.h, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.h, z14.h, z4.h\n" + "ld1h z18.h, p0/z, [x8, #5, MUL VL]\n" + "fmax z19.h, p2/m, z19.h, z1.h\n" + "fmax z20.h, p0/m, z20.h, z1.h\n" + "fmax z13.h, p1/m, z13.h, z1.h\n" + "fmin z14.h, p2/m, z14.h, z0.h\n" + "st1h z19.h, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z15.h, z15.h, z2.h\n" + "ld1h z19.h, p1/z, [x8, #6, MUL VL]\n" + "fadd z16.h, z16.h, z3.h\n" + "addvl %[outptr4], %[outptr4], #3\n" + "fmax z14.h, p2/m, z14.h, z1.h\n" + "st1h z20.h, p0, [%[outptr5]]\n" + "fmin z15.h, p0/m, z15.h, z0.h\n" + "ld1h z20.h, p2/z, [x8, #7, MUL VL]\n" + "fmin z16.h, p1/m, z16.h, z0.h\n" + "fadd z17.h, z17.h, z4.h\n" + "st1h z13.h, p1, [%[outptr5], #1, MUL VL]\n" + "fadd z18.h, z18.h, z2.h\n" + "fmax z15.h, p0/m, z15.h, z1.h\n" + "fmax z16.h, p1/m, z16.h, z1.h\n" + "st1h z14.h, p2, [%[outptr5], #2, MUL VL]\n" + "fmin z17.h, p2/m, z17.h, z0.h\n" + "addvl %[outptr5], %[outptr5], #3\n" + "fmin z18.h, p0/m, z18.h, z0.h\n" + "st1h z15.h, p0, [%[outptr6]]\n" + "fadd z19.h, z19.h, z3.h\n" + "fmax z17.h, p2/m, z17.h, z1.h\n" + "fadd z20.h, z20.h, z4.h\n" + "st1h z16.h, p1, [%[outptr6], #1, MUL VL]\n" + "fmax z18.h, p0/m, z18.h, z1.h\n" + "fmin z19.h, p1/m, z19.h, z0.h\n" + "fmin z20.h, p2/m, z20.h, z0.h\n" + "st1h z17.h, p2, [%[outptr6], #2, MUL VL]\n" + "addvl %[outptr6], %[outptr6], #3\n" + "fmax z19.h, p1/m, z19.h, z1.h\n" + "fmax z20.h, p2/m, z20.h, z1.h\n" + "st1h z18.h, p0, [%[outptr7]]\n" + "st1h z19.h, p1, [%[outptr7], #1, MUL VL]\n" + "st1h z20.h, p2, [%[outptr7], #2, MUL VL]\n" + "addvl %[outptr7], %[outptr7], #3\n" + : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), + [inptr] "+r" (inptr), [p] "+r" (p) + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" + ); + } + break; + + + } + } + } + } +} + +#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp deleted file mode 100644 index 7479c8d77c..0000000000 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_2VLx8.hpp +++ /dev/null @@ -1,1208 +0,0 @@ -/* - * Copyright (c) 2018 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -inline void MergeResults<2, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) -{ - const float *inptr = in; - - for (int y=y0; y())) { - if (beta==0.0f) - { - switch(height) { - case 1: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 2: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 3: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 4: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 5: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 6: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PSTL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 7: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-5, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PSTL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PSTL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - default: - case 8: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z7.s, p0/z, [x8, #-2, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x40]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x40]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PSTL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PSTL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "prfm PSTL1KEEP, [%[outptr7], #0x40]\n" - "addvl %[outptr7], %[outptr7], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - - } - } - else - { - switch(height) { - case 1: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 2: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 3: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 4: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 5: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 6: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PLDL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - case 7: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PLDL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PLDL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - default: - case 8: - { - long w = xmax - i; - long p = 0; - /* Optimized routine to copy an entire block */ - __asm __volatile ( - "mov z2.s, %s[alpha]\n" - "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" - "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x100]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x140]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z11.s, p0/z, [%[outptr7]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "addvl %[outptr1], %[outptr1], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x40]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "addvl %[outptr2], %[outptr2], #2\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x40]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "addvl %[outptr3], %[outptr3], #2\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x40]\n" - "addvl %[outptr4], %[outptr4], #2\n" - "prfm PLDL1KEEP, [%[outptr5], #0x40]\n" - "addvl %[outptr5], %[outptr5], #2\n" - "prfm PLDL1KEEP, [%[outptr6], #0x40]\n" - "addvl %[outptr6], %[outptr6], #2\n" - "prfm PLDL1KEEP, [%[outptr7], #0x40]\n" - "addvl %[outptr7], %[outptr7], #2\n" - "1:\n" - "addvl %[inptr], %[inptr], #16\n" - : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), - [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" - ); - } - break; - - - } - } - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp index 4b51066ebe..f7feec4d83 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp @@ -26,11 +26,33 @@ #ifdef __ARM_FEATURE_SVE template<> -inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float alpha, const float beta) +void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) { const float *inptr = in; + float nullbias[192] = { 0 }; + float minval = - std::numeric_limits::infinity(); + float maxval = std::numeric_limits::infinity(); - for (int y=y0; y(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length() * sizeof(float))); + } + + for (int y=y0; y(float *out, const float *in, const int ldou const int height = ymax - y; - for (int i=x0; i())) { - if (beta==0.0f) + for (int i=x0; i())) + { + if (append) { - switch(height) { + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "fadd z12.s, z12.s, z4.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -91,47 +120,61 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "fadd z14.s, z14.s, z6.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -142,59 +185,82 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z16.s, z16.s, z8.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -205,70 +271,102 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z12.s, z12.s, z4.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -279,82 +377,123 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z14.s, z14.s, z6.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z15.s, z15.s, z7.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z16.s, z16.s, z8.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "addvl %[outptr4], %[outptr4], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -365,94 +504,144 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z10.s, p1/z, [x8]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "fmin z10.s, p1/m, z10.s, z0.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmin z11.s, p2/m, z11.s, z0.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z10.s, p1/m, z10.s, z1.s\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z11.s, p2/m, z11.s, z1.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "addvl %[outptr5], %[outptr5], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -463,106 +652,165 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #3, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z10.s, p1/z, [x8]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "fmin z10.s, p1/m, z10.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z10.s, p1/m, z10.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #4, MUL VL]\n" + "fmin z11.s, p2/m, z11.s, z0.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fadd z13.s, z13.s, z5.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "fmax z11.s, p2/m, z11.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmin z12.s, p0/m, z12.s, z0.s\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "fmax z12.s, p0/m, z12.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "addvl %[outptr5], %[outptr5], #3\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -574,117 +822,185 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z7.s, p0/z, [x8, #5, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-3, MUL VL]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [x8]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #3, MUL VL]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #6, MUL VL]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmin z10.s, p0/m, z10.s, z0.s\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fmax z10.s, p0/m, z10.s, z1.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "fmin z11.s, p1/m, z11.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "fadd z14.s, z14.s, z6.s\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fmax z11.s, p1/m, z11.s, z1.s\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "fmin z12.s, p2/m, z12.s, z0.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "fmax z12.s, p2/m, z12.s, z1.s\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmin z10.s, p2/m, z10.s, z0.s\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z10.s, p2/m, z10.s, z1.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "fmin z11.s, p0/m, z11.s, z0.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z12.s, p1/m, z12.s, z0.s\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z11.s, p0/m, z11.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "fmax z12.s, p1/m, z12.s, z1.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z10.s, p1/z, [x8]\n" + "fadd z15.s, z15.s, z7.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "fadd z16.s, z16.s, z8.s\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "fadd z10.s, z10.s, z2.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "fmin z10.s, p1/m, z10.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "fadd z11.s, z11.s, z3.s\n" + "ld1w z7.s, p0/z, [%[outptr7]]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z12.s, z12.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n" + "fmax z10.s, p1/m, z10.s, z1.s\n" + "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n" + "fmin z11.s, p2/m, z11.s, z0.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z13.s, z13.s, z5.s\n" + "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n" + "fmin z12.s, p0/m, z12.s, z0.s\n" + "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n" + "fadd z14.s, z14.s, z6.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "fmax z11.s, p2/m, z11.s, z1.s\n" + "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" + "fmax z12.s, p0/m, z12.s, z1.s\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z6.s, p0/z, [x8, #4, MUL VL]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "fadd z15.s, z15.s, z7.s\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "fadd z16.s, z16.s, z8.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z10.s, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [x8, #7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmul z11.s, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "fadd z17.s, z17.s, z9.s\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.s, p1/m, z16.s, z0.s\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" + "fmax z15.s, p0/m, z15.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmax z16.s, p1/m, z16.s, z1.s\n" + "st1w z15.s, p0, [%[outptr7]]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" "addvl %[outptr5], %[outptr5], #3\n" + "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" "addvl %[outptr6], %[outptr6], #3\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -694,48 +1010,54 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou } else { - switch(height) { + const float *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z13.s, z13.s, z2.s\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "fadd z15.s, z15.s, z4.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -746,59 +1068,58 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "fadd z15.s, z15.s, z4.s\n" + "fadd z16.s, z16.s, z2.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fadd z17.s, z17.s, z3.s\n" + "fadd z18.s, z18.s, z4.s\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -809,77 +1130,76 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "fadd z20.s, z20.s, z3.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.s, z13.s, z4.s\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -890,94 +1210,93 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fadd z13.s, z13.s, z4.s\n" + "fadd z14.s, z14.s, z2.s\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "fadd z15.s, z15.s, z3.s\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -988,112 +1307,111 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z9.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z16.s, z16.s, z4.s\n" + "fadd z17.s, z17.s, z2.s\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "fadd z18.s, z18.s, z3.s\n" + "fadd z19.s, z19.s, z4.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1104,130 +1422,129 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "fadd z18.s, z18.s, z3.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z19.s, z19.s, z4.s\n" + "fadd z20.s, z20.s, z2.s\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmin z20.s, p0/m, z20.s, z0.s\n" + "fadd z13.s, z13.s, z3.s\n" + "fadd z14.s, z14.s, z4.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "fmax z20.s, p0/m, z20.s, z1.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1238,148 +1555,147 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z11.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z10.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #1, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr6], #2, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.s, z18.s, z3.s\n" + "fadd z19.s, z19.s, z4.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.s, z20.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.s, z13.s, z3.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.s, p0/m, z20.s, z0.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.s, z14.s, z4.s\n" + "fadd z15.s, z15.s, z2.s\n" + "fmax z20.s, p0/m, z20.s, z1.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" + "fadd z16.s, z16.s, z3.s\n" + "fadd z17.s, z17.s, z4.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "fmax z15.s, p0/m, z15.s, z1.s\n" + "fmin z16.s, p1/m, z16.s, z0.s\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "fmax z16.s, p1/m, z16.s, z1.s\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1391,165 +1707,164 @@ inline void MergeResults<3, 8, true>(float *out, const float *in, const int ldou long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" + "mov z0.s, %s[maxval]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" + "mov z1.s, %s[minval]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0]]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr]]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z9.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #3, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z10.s, p0/z, [%[outptr2]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z11.s, p0/z, [%[outptr3]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z8.s, p0/z, [%[outptr4]]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-4, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "ld1w z9.s, p0/z, [%[outptr5]]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #-1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "ld1w z10.s, p0/z, [%[outptr6]]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #2, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "ld1w z11.s, p0/z, [%[outptr7]]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #1, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #4, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [%[inptr], #7, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-3, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #3, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #6, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "ld1w z8.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [%[inptr], #2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr1], #2, MUL VL]\n" + "fadd z18.s, z18.s, z4.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z19.s, z19.s, z2.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [%[inptr], #5, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #-8, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "fmin z19.s, p0/m, z19.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fadd z13.s, z13.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" + "fadd z14.s, z14.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #-5, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fmul z8.s, z8.s, z3.s\n" - "ld1w z4.s, p0/z, [x8, #-2, MUL VL]\n" - "fmla z8.s, p0/m, z4.s, z2.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "fmul z9.s, z9.s, z3.s\n" - "ld1w z5.s, p0/z, [x8, #1, MUL VL]\n" - "fmla z9.s, p0/m, z5.s, z2.s\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[outptr6], #2, MUL VL]\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "fmin z14.s, p0/m, z14.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fadd z15.s, z15.s, z3.s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fadd z16.s, z16.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" + "fadd z17.s, z17.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "addvl %[outptr2], %[outptr2], #3\n" - "fmul z10.s, z10.s, z3.s\n" - "ld1w z6.s, p0/z, [x8, #4, MUL VL]\n" - "fmla z10.s, p0/m, z6.s, z2.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[outptr7], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "fmul z11.s, z11.s, z3.s\n" - "ld1w z7.s, p0/z, [x8, #7, MUL VL]\n" - "fmla z11.s, p0/m, z7.s, z2.s\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "fmin z17.s, p0/m, z17.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fadd z18.s, z18.s, z3.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fmax z17.s, p0/m, z17.s, z1.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" + "fadd z19.s, z19.s, z4.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" + "fadd z20.s, z20.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "fadd z13.s, z13.s, z3.s\n" "addvl %[outptr3], %[outptr3], #3\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "fmin z20.s, p0/m, z20.s, z0.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.s, z14.s, z4.s\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "fmax z20.s, p0/m, z20.s, z1.s\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" + "fadd z15.s, z15.s, z2.s\n" + "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" + "fadd z16.s, z16.s, z3.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" + "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "fmin z16.s, p1/m, z16.s, z0.s\n" + "fadd z17.s, z17.s, z4.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "fadd z18.s, z18.s, z2.s\n" + "fmax z15.s, p0/m, z15.s, z1.s\n" + "fmax z16.s, p1/m, z16.s, z1.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" "addvl %[outptr5], %[outptr5], #3\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "fmin z18.s, p0/m, z18.s, z0.s\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "fadd z19.s, z19.s, z3.s\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" + "fadd z20.s, z20.s, z4.s\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "fmax z18.s, p0/m, z18.s, z1.s\n" + "fmin z19.s, p1/m, z19.s, z0.s\n" + "fmin z20.s, p2/m, z20.s, z0.s\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "fmax z19.s, p1/m, z19.s, z1.s\n" + "fmax z20.s, p2/m, z20.s, z1.s\n" + "st1w z18.s, p0, [%[outptr7]]\n" + "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr), [minval] "w" (minval), [maxval] "w" (maxval) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp index 2ea38a78df..03a28241fd 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_s32_3VLx8.hpp @@ -26,11 +26,20 @@ #ifdef __ARM_FEATURE_SVE template<> -inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t alpha, const int32_t beta) +void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation act, bool append) { + UNUSED(act); + const int32_t *inptr = in; + int32_t nullbias[192] = { 0 }; + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length() * sizeof(int32_t))); + } - for (int y=y0; y(int32_t *out, const int32_t *in, const int const int height = ymax - y; - for (int i=x0; i())) { - if (beta==0) + for (int i=x0; i())) + { + if (append) { - switch(height) { + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -88,41 +99,47 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -133,50 +150,62 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -187,58 +216,76 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -249,67 +296,91 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "add z16.s, z16.s, z8.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -320,76 +391,106 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -400,85 +501,121 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -490,93 +627,135 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "ld1w z7.s, p0/z, [%[outptr7]]\n" "addvl %[outptr5], %[outptr5], #3\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n" + "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n" + "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n" + "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p0, [%[outptr7]]\n" + "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -586,45 +765,46 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int } else { - switch(height) { + const int32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z13.s, z13.s, z2.s\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -635,53 +815,44 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z4.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z16.s, z16.s, z2.s\n" + "add z17.s, z17.s, z3.s\n" + "add z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -692,68 +863,56 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z20.s, z20.s, z3.s\n" + "add z13.s, z13.s, z4.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -764,82 +923,67 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z16.s, z16.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -850,97 +994,79 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -951,112 +1077,91 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z7.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1067,127 +1172,103 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z7.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z4.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1199,141 +1280,114 @@ inline void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr7]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8]\n" - "ld1w z7.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z6.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z7.s, p0/z, [%[outptr7], #2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z18.s, z18.s, z2.s\n" + "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "add z19.s, z19.s, z3.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z20.s, z20.s, z4.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "st1w z18.s, p0, [%[outptr7]]\n" + "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp index eb684e2118..67a6eb32bb 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_u32_3VLx8.hpp @@ -26,11 +26,20 @@ #ifdef __ARM_FEATURE_SVE template<> -inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t alpha, const uint32_t beta) +void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const uint32_t *bias, Activation act, bool append) { + UNUSED(act); + const uint32_t *inptr = in; + uint32_t nullbias[192] = { 0 }; + + if (!append && !bias) + { + memset(nullbias, 0, (3 * get_vector_length() * sizeof(uint32_t))); + } - for (int y=y0; y(uint32_t *out, const uint32_t *in, const in const int height = ymax - y; - for (int i=x0; i())) { - if (beta==0u) + for (int i=x0; i())) + { + if (append) { - switch(height) { + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -88,41 +99,47 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -133,50 +150,62 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -187,58 +216,76 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -249,67 +296,91 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "add z16.s, z16.s, z8.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -320,76 +391,106 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -400,85 +501,121 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -490,93 +627,135 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[outptr0]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z10.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" + "ld1w z5.s, p0/z, [%[outptr1]]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n" + "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z10.s, p0, [%[outptr0]]\n" + "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n" + "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" + "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[outptr2]]\n" "addvl %[outptr0], %[outptr0], #3\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p0, [%[outptr1]]\n" + "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n" + "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n" + "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n" + "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "ld1w z3.s, p0/z, [%[outptr3]]\n" + "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p0, [%[outptr2]]\n" + "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n" + "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n" + "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n" + "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p0/z, [%[outptr4]]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n" + "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n" + "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n" + "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n" + "ld1w z9.s, p0/z, [%[outptr5]]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n" + "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n" + "ld1w z10.s, p1/z, [x8]\n" + "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n" + "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n" + "add z10.s, z10.s, z2.s\n" + "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z4.s, p0/z, [%[outptr6]]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z11.s, z11.s, z3.s\n" + "st1w z17.s, p0, [%[outptr5]]\n" + "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n" + "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n" + "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z12.s, z12.s, z4.s\n" + "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n" + "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n" + "add z13.s, z13.s, z5.s\n" + "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n" + "ld1w z7.s, p0/z, [%[outptr7]]\n" "addvl %[outptr5], %[outptr5], #3\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "add z14.s, z14.s, z6.s\n" + "st1w z12.s, p0, [%[outptr6]]\n" + "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n" + "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n" + "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n" + "add z15.s, z15.s, z7.s\n" + "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n" + "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n" + "add z16.s, z16.s, z8.s\n" + "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "add z17.s, z17.s, z9.s\n" + "st1w z15.s, p0, [%[outptr7]]\n" + "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -586,45 +765,46 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in } else { - switch(height) { + const uint32_t *biasptr = nullbias; + if (bias) + { + biasptr = bias + i; + } + + switch(height) + { case 1: { long w = xmax - i; long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "addvl %[outptr0], %[outptr0], #3\n" - "1:\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "add z13.s, z13.s, z2.s\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -635,53 +815,44 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z4.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z16.s, z16.s, z2.s\n" + "add z17.s, z17.s, z3.s\n" + "add z18.s, z18.s, z4.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -692,68 +863,56 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" + "add z20.s, z20.s, z3.s\n" + "add z13.s, z13.s, z4.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -764,82 +923,67 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "add z16.s, z16.s, z4.s\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -850,97 +994,79 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z5.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-3, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr2], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr4], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -951,112 +1077,91 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z6.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z10.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z8.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z10.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-3, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z7.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8]\n" - "st1w z8.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr3], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr5], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "add z18.s, z18.s, z3.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1067,127 +1172,103 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z7.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z11.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z9.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-3, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z4.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8]\n" - "st1w z9.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z10.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z11.s, p0, [%[outptr4], #1, MUL VL]\n" - "st1w z8.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr6], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z6.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z10.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z7.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z8.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z6.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "add z19.s, z19.s, z4.s\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z7.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z4.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "st1w z10.s, p0, [%[outptr4], #2, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z8.s, p0/z, [x8, #4, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr5], #2, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "st1w z8.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; @@ -1199,141 +1280,114 @@ inline void MergeResults<3, 8, true>(uint32_t *out, const uint32_t *in, const in long p = 0; /* Optimized routine to copy an entire block */ __asm __volatile ( - "mov z2.s, %s[alpha]\n" "addvl x8, %[inptr], #16\n" - "mov z3.s, %s[beta]\n" "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" "incw %[p], all, mul #1\n" "prfm PLDL1KEEP, [%[inptr], #0x180]\n" - "ld1w z4.s, p0/z, [%[outptr0]]\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "ld1w z8.s, p0/z, [%[inptr]]\n" - "ld1w z5.s, p0/z, [%[outptr1]]\n" - "ld1w z9.s, p0/z, [%[inptr], #3, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #6, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4]]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0]]\n" - "ld1w z8.s, p0/z, [x8, #-4, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5]]\n" - "ld1w z6.s, p0/z, [%[outptr6]]\n" - "st1w z9.s, p0, [%[outptr1]]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8, #-1, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr7]]\n" - "st1w z10.s, p0, [%[outptr2]]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #2, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3]]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #5, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4]]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5]]\n" - "st1w z10.s, p0, [%[outptr6]]\n" - "st1w z11.s, p0, [%[outptr7]]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" + "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" + "ld1w z2.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[p], %[w]\n" + "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" "incw %[p], all, mul #1\n" + "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z4.s, p0/z, [%[outptr0], #1, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" - "ld1w z8.s, p0/z, [%[inptr], #1, MUL VL]\n" - "ld1w z5.s, p0/z, [%[outptr1], #1, MUL VL]\n" - "ld1w z9.s, p0/z, [%[inptr], #4, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z10.s, p0/z, [%[inptr], #7, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z11.s, p0/z, [x8, #-6, MUL VL]\n" - "ld1w z4.s, p0/z, [%[outptr4], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "st1w z8.s, p0, [%[outptr0], #1, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-3, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "ld1w z5.s, p0/z, [%[outptr5], #1, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr6], #1, MUL VL]\n" - "st1w z9.s, p0, [%[outptr1], #1, MUL VL]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [x8]\n" - "ld1w z7.s, p0/z, [%[outptr7], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr2], #1, MUL VL]\n" - "add z9.s, z9.s, z5.s\n" - "ld1w z10.s, p0/z, [x8, #3, MUL VL]\n" - "st1w z11.s, p0, [%[outptr3], #1, MUL VL]\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #6, MUL VL]\n" - "st1w z8.s, p0, [%[outptr4], #1, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z9.s, p0, [%[outptr5], #1, MUL VL]\n" - "st1w z10.s, p0, [%[outptr6], #1, MUL VL]\n" - "st1w z11.s, p0, [%[outptr7], #1, MUL VL]\n" - "whilelt p0.s, %[p], %[w]\n" - "b.none 1f\n" - "prfm PLDL1KEEP, [%[outptr0], #0x60]\n" - "prfm PLDL1KEEP, [%[outptr1], #0x60]\n" - "ld1w z4.s, p0/z, [%[outptr0], #2, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" + "whilelt p2.s, %[p], %[w]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" + "add z13.s, z13.s, z2.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "ld1w z8.s, p0/z, [%[inptr], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr2], #0x60]\n" - "ld1w z5.s, p0/z, [%[outptr1], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr3], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "ld1w z9.s, p0/z, [%[inptr], #5, MUL VL]\n" - "ld1w z6.s, p0/z, [%[outptr2], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr4], #0x60]\n" - "ld1w z10.s, p0/z, [x8, #-8, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr5], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr0], #2, MUL VL]\n" - "ld1w z7.s, p0/z, [%[outptr3], #2, MUL VL]\n" + "add z14.s, z14.s, z3.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "add z15.s, z15.s, z4.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "add z16.s, z16.s, z2.s\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "add z17.s, z17.s, z3.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "add z18.s, z18.s, z4.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "add z19.s, z19.s, z2.s\n" + "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" + "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "add z20.s, z20.s, z3.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "add z13.s, z13.s, z4.s\n" + "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" + "add z14.s, z14.s, z2.s\n" "addvl %[outptr0], %[outptr0], #3\n" - "add z10.s, z10.s, z6.s\n" - "st1w z9.s, p0, [%[outptr1], #2, MUL VL]\n" - "ld1w z11.s, p0/z, [x8, #-5, MUL VL]\n" + "st1w z16.s, p0, [%[outptr1]]\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "add z15.s, z15.s, z3.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "add z16.s, z16.s, z4.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" - "ld1w z4.s, p0/z, [%[outptr4], #2, MUL VL]\n" + "add z17.s, z17.s, z2.s\n" + "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr2], #2, MUL VL]\n" - "ld1w z8.s, p0/z, [x8, #-2, MUL VL]\n" + "st1w z19.s, p0, [%[outptr2]]\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "add z18.s, z18.s, z3.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "addvl %[inptr], %[inptr], #24\n" + "add z19.s, z19.s, z4.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" - "ld1w z5.s, p0/z, [%[outptr5], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr6], #0x60]\n" - "add z8.s, z8.s, z4.s\n" - "st1w z11.s, p0, [%[outptr3], #2, MUL VL]\n" - "ld1w z9.s, p0/z, [x8, #1, MUL VL]\n" + "add z20.s, z20.s, z2.s\n" + "ld1w z13.s, p1/z, [x8]\n" + "st1w z14.s, p0, [%[outptr3]]\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "add z13.s, z13.s, z3.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "add z14.s, z14.s, z4.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" - "ld1w z6.s, p0/z, [%[outptr6], #2, MUL VL]\n" - "prfm PLDL1KEEP, [%[outptr7], #0x60]\n" - "add z9.s, z9.s, z5.s\n" - "st1w z8.s, p0, [%[outptr4], #2, MUL VL]\n" - "ld1w z10.s, p0/z, [x8, #4, MUL VL]\n" + "add z15.s, z15.s, z2.s\n" + "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" + "st1w z17.s, p0, [%[outptr4]]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "add z16.s, z16.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "add z17.s, z17.s, z4.s\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" - "ld1w z7.s, p0/z, [%[outptr7], #2, MUL VL]\n" - "st1w z9.s, p0, [%[outptr5], #2, MUL VL]\n" + "add z18.s, z18.s, z2.s\n" + "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" + "st1w z20.s, p0, [%[outptr5]]\n" + "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "add z19.s, z19.s, z3.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "add z20.s, z20.s, z4.s\n" + "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" - "add z10.s, z10.s, z6.s\n" - "ld1w z11.s, p0/z, [x8, #7, MUL VL]\n" - "add z11.s, z11.s, z7.s\n" - "st1w z10.s, p0, [%[outptr6], #2, MUL VL]\n" + "st1w z15.s, p0, [%[outptr6]]\n" + "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" + "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" - "st1w z11.s, p0, [%[outptr7], #2, MUL VL]\n" + "st1w z18.s, p0, [%[outptr7]]\n" + "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n" + "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n" "addvl %[outptr7], %[outptr7], #3\n" - "1:\n" - "addvl %[inptr], %[inptr], #24\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), [inptr] "+r" (inptr), [p] "+r" (p) - : [alpha] "w" (alpha), [beta] "w" (beta), [w] "r" (w) - : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "memory", "cc" + : [w] "r" (w), [biasptr] "r" (biasptr) + : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc" ); } break; diff --git a/src/core/NEON/kernels/arm_gemm/misc.cpp b/src/core/NEON/kernels/arm_gemm/misc.cpp index 7b345e2e98..6758a88c65 100644 --- a/src/core/NEON/kernels/arm_gemm/misc.cpp +++ b/src/core/NEON/kernels/arm_gemm/misc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2017-2019 ARM Limited. * * SPDX-License-Identifier: MIT * diff --git a/src/core/NEON/kernels/arm_gemm/profiler.hpp b/src/core/NEON/kernels/arm_gemm/profiler.hpp deleted file mode 100644 index 1b944c4ccd..0000000000 --- a/src/core/NEON/kernels/arm_gemm/profiler.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef CYCLE_PROFILING - -#include "../perf.h" - -#ifndef NO_MULTI_THREADING -#include -#endif - -namespace arm_gemm { - -#ifndef NO_MULTI_THREADING -extern std::mutex report_mutex; -#endif - -class profiler { -private: - static const int maxevents = 100000; - unsigned long times[maxevents] = { }; - unsigned long units[maxevents] = { }; - int events[maxevents] = { }; - int currentevent=0; - int countfd=0; - - class ScopedProfilerClass { - private: - profiler &_parent; - bool legal=false; - - public: - ScopedProfilerClass(profiler &prof, int i, unsigned long u) : _parent(prof) { - if (prof.currentevent==maxevents) - return; - - prof.events[prof.currentevent]=i; - prof.units[prof.currentevent]=u; - legal=true; - start_counter(prof.countfd); - } - - ~ScopedProfilerClass() { - if (!legal) return; - - long long cycs = stop_counter(_parent.countfd); - _parent.times[_parent.currentevent++] = cycs; - } - }; - -public: - profiler() { - countfd=open_cycle_counter(); - } - - ~profiler() { - close(countfd); - int tots[5]; - unsigned long counts[5]; - unsigned long tunits[5]; - const char * descs[] = { "Prepare A", "Prepare B", "Kernel", "Merge" }; - - for (int i=1; i<5; i++) { - tots[i] = 0; - counts[i] = 0; - tunits[i] = 0; - } - - for (int i=0; i lock(report_mutex); - printf("Profiled events (cpu %d):\n", sched_getcpu()); -#endif - - printf("%20s %9s %9s %9s %12s %9s\n", "", "Events", "Total", "Average", "Bytes/MACs", "Per cycle"); - for (int i=1; i<5; i++) { - printf("%20s: %9d %9ld %9ld %12lu %9.2f\n",descs[i-1],tots[i],counts[i],counts[i]/tots[i],tunits[i],(float)tunits[i]/counts[i]); - } - } - - template - void operator() (int i, unsigned long u, T func) { - if (currentevent==maxevents) { - func(); - } else { - events[currentevent] = i; - units[currentevent] = u; - start_counter(countfd); - func(); - long long cycs = stop_counter(countfd); - times[currentevent++] = cycs; - } - } - - ScopedProfilerClass ScopedProfiler(int i, unsigned long u) { - return ScopedProfilerClass(*this, i, u); - } -}; - -#endif // CYCLE_PROFILING - -} // namespace arm_gemm - -#define PROFILE_PREPA 1 -#define PROFILE_PREPB 2 -#define PROFILE_KERNEL 3 -#define PROFILE_MERGE 4 diff --git a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp index 5e67bc9c0e..188dd0b06d 100644 --- a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp +++ b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp @@ -41,7 +41,7 @@ private: int32_t *_row_sums = nullptr; int32_t *_col_sums = nullptr; ARequantizeLayer32 _params; - GemmArgs _args; + GemmArgs _args; barrier _barrier; void *working_space = nullptr; @@ -80,12 +80,13 @@ private: /* Use the first part of our working space for the subgemm result, pass the operand details straight through. */ _subgemm->set_arrays(this->_Aptr, this->_lda, this->_A_batch_stride, this->_A_multi_stride, this->_Bptr, this->_ldb, this->_B_multi_stride, - reinterpret_cast(working_space), _args._Nsize, (_args._Nsize * _args._Msize), (_args._Nsize * _args._Msize * _args._nbatches)); + reinterpret_cast(working_space), _args._Nsize, (_args._Nsize * _args._Msize), (_args._Nsize * _args._Msize * _args._nbatches), + nullptr, 0); } void col_sums_pretransposed(const To *B, const int ldb, const int B_multi_stride) { for (unsigned int multi=0; multi<_args._nmulti; multi++) { - compute_col_sums(_params, _args._Nsize, _args._Ksize, B + (multi * B_multi_stride), ldb, _col_sums + (multi * _args._Nsize), _args._Ksize, 0); + compute_col_sums(_params, _args._Nsize, _args._Ksize, B + (multi * B_multi_stride), ldb, _col_sums + (multi * _args._Nsize), _args._Ksize, multi, 0); } } @@ -94,7 +95,7 @@ private: unsigned int last_col = ((threadid + 1) * _args._Nsize) / _args._maxthreads; for (unsigned int multi=0; multi<_args._nmulti; multi++) { - compute_col_sums(_params, (last_col - first_col), _args._Ksize, this->_Bptr + (multi * this->_B_multi_stride) + first_col, this->_ldb, _col_sums + (multi * _args._Nsize) + first_col, _args._Ksize, first_col); + compute_col_sums(_params, (last_col - first_col), _args._Ksize, this->_Bptr + (multi * this->_B_multi_stride) + first_col, this->_ldb, _col_sums + (multi * _args._Nsize) + first_col, _args._Ksize, multi, first_col); } } @@ -121,8 +122,11 @@ private: public: - QuantizeWrapper(const GemmArgs &args, const ARequantizeLayer32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) { - GemmArgs newargs = GemmArgs(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._trA, args._trB, 1, 0, args._maxthreads, args._pretransposed_hint, nullptr); + QuantizeWrapper(const QuantizeWrapper &) = delete; + QuantizeWrapper operator=(const QuantizeWrapper &) = delete; + + QuantizeWrapper(const GemmArgs &args, const ARequantizeLayer32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) { + GemmArgs newargs = GemmArgs(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._trA, args._trB, Activation(), args._maxthreads, args._pretransposed_hint, nullptr); _subgemm = gemm(newargs); if (_subgemm == nullptr) { @@ -134,15 +138,11 @@ public: } } - QuantizeWrapper(const QuantizeWrapper &) = delete; - QuantizeWrapper &operator=(const QuantizeWrapper &) = delete; - QuantizeWrapper(QuantizeWrapper &&) = default; - QuantizeWrapper &operator=(QuantizeWrapper &&) = default; - void set_arrays(const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, const int B_multi_stride, - Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride) override { - GemmCommon::set_arrays(A, lda, A_batch_stride, A_multi_stride, B, ldb, B_multi_stride, C, ldc, C_batch_stride, C_multi_stride); + Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, + const Tr *bias, const int bias_multi_stride) override { + GemmCommon::set_arrays(A, lda, A_batch_stride, A_multi_stride, B, ldb, B_multi_stride, C, ldc, C_batch_stride, C_multi_stride, bias, bias_multi_stride); arrays_set = true; set_child_arrays(); @@ -232,8 +232,9 @@ public: _col_sums = reinterpret_cast(buffer); } - void set_quantized_bias(const int32_t *bias) override { + void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override { _params.bias = bias; + _params.bias_multi_stride = bias_multi_stride; } }; diff --git a/src/core/NEON/kernels/arm_gemm/quantized.cpp b/src/core/NEON/kernels/arm_gemm/quantized.cpp index 28f01bd252..bffb7ddcb3 100644 --- a/src/core/NEON/kernels/arm_gemm/quantized.cpp +++ b/src/core/NEON/kernels/arm_gemm/quantized.cpp @@ -495,10 +495,10 @@ namespace { * We could do 64 adds in the signed case, but that * optimization is not worth the complexity. */ - if (i > 0 && ((i & 31) == 0)) { - finalsums[r] = vpadalq_s16(finalsums[r], sums[r]); - sums[r] = vdupq_n_s16(0); - } + if (i > 0 && ((i & 31) == 0)) { + finalsums[r] = vpadalq_s16(finalsums[r], sums[r]); + sums[r] = vdupq_n_s16(0); + } sums[r] = accumulate_16(input + (r * in_stride) + (i * 16), sums[r]); } } @@ -526,6 +526,7 @@ namespace { * that the terms can simply be added in the requantize code. * */ switch (rows) { + default: case 1: /* If we only have one output, just use ADDV. Multiply * the offset into all four components separately so it @@ -567,8 +568,6 @@ namespace { vst1q_s32(row_bias, t0); break; - default: - break; } } @@ -736,12 +735,11 @@ inline void add_block(const int8_t *input, unsigned int in_stride, int32_t *outp } } - /* "first_col" parameter is used to offset the read into the qp.bias array, * in cases where we are not computing the first columns of the output (i.e. * in multithreaded cases where we divide columns across threads) */ template -void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int first_col) { +void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col) { memset(reinterpret_cast(col_bias), 0, width * sizeof(int32_t)); for (unsigned int row=0; row(input + row * in_stride + col, in_stride, col_bias + col); break; @@ -767,8 +766,6 @@ void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned case 4: add_block<4>(input + row * in_stride + col, in_stride, col_bias + col); break; - default: - break; } } else { for (; col void compute_col_sums(const ARequantizeLayer32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, - unsigned int first_col); + unsigned int multi, unsigned int first_col); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp index 2c661e7fa9..c3c1e8d930 100644 --- a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp +++ b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp @@ -64,8 +64,8 @@ public: } template - void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut alpha, const TOut beta) { - MergeResults(out, in, stride, y0, ymax, x0, xmax, alpha, beta); + void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool append) { + MergeResults(out, in, stride, y0, ymax, x0, xmax, bias, act, append); } }; diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp index b7323ebaea..6b64e5e36c 100644 --- a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp +++ b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp @@ -63,8 +63,8 @@ public: } template - void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut alpha, const TOut beta) { - MergeResults(out, in, stride, y0, ymax, x0, xmax, alpha, beta); + void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool append) { + MergeResults(out, in, stride, y0, ymax, x0, xmax, bias, act, append); } }; diff --git a/src/core/NEON/kernels/arm_gemm/transform.hpp b/src/core/NEON/kernels/arm_gemm/transform.hpp index d790d566b6..bdae90300b 100644 --- a/src/core/NEON/kernels/arm_gemm/transform.hpp +++ b/src/core/NEON/kernels/arm_gemm/transform.hpp @@ -84,7 +84,6 @@ struct TransformImpl { *out++ = static_cast(0); } } - // "row" tail - row is out of range so fill with zeros always. TOut zeroval = static_cast(0); int pads = blank_rows * (fill_cols + blank_cols); diff --git a/src/core/NEON/kernels/assembly/Helpers.cpp b/src/core/NEON/kernels/assembly/Helpers.cpp index 3d8d66d7fc..93ea6c8d5e 100644 --- a/src/core/NEON/kernels/assembly/Helpers.cpp +++ b/src/core/NEON/kernels/assembly/Helpers.cpp @@ -24,16 +24,13 @@ #include "arm_compute/core/NEON/kernels/assembly/Helpers.h" -#include "arm_compute/core/NEON/kernels/assembly/arm_gemm.hpp" - namespace arm_compute { arm_gemm::KernelDescription get_gemm_info(DataType input_type, const CPUInfo &ci, const unsigned int num_threads, const INEGEMMWrapperKernel::Params &p, - float alpha, - float beta, + arm_gemm::Activation activation, bool pretranspose_hint) { switch(input_type) @@ -42,25 +39,25 @@ arm_gemm::KernelDescription get_gemm_info(DataType in case DataType::QASYMM8: case DataType::U8: { - arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method(args); } case DataType::S8: { - arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method(args); } #endif // __aarch64__ #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC case DataType::F16: { - arm_gemm::GemmArgs<__fp16> args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method<__fp16, __fp16>(args); } #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ case DataType::F32: { - arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, pretranspose_hint); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, pretranspose_hint); return arm_gemm::get_gemm_method(args); } default: diff --git a/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h b/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h deleted file mode 100644 index 6e30148b5d..0000000000 --- a/src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ -#define __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ - -#include "../arm_gemm/utils.hpp" -#include "arm_gemm.hpp" - -#include "../arm_gemm/mergeresults.hpp" -#include "../arm_gemm/transform.hpp" - -#include "../arm_gemm/kernels/a32_sgemm_8x6.hpp" -#include "../arm_gemm/kernels/a64_gemm_s8_12x8.hpp" -#include "../arm_gemm/kernels/a64_gemm_s8_4x4.hpp" -#include "../arm_gemm/kernels/a64_gemm_u8_12x8.hpp" -#include "../arm_gemm/kernels/a64_gemm_u8_4x4.hpp" -#include "../arm_gemm/kernels/a64_hgemm_24x8.hpp" -#include "../arm_gemm/kernels/a64_sgemm_12x8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp" -#include "../arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp" - -namespace arm_compute -{ -namespace detail -{ -/** GEMM Interleaved Strategy interface */ -class IInterleavedStrategy -{ -public: - /** Virtual Destructor */ - virtual ~IInterleavedStrategy() = default; - /** Return output height of the interleaved strategy - * - * @return Output height of strategy - */ - virtual unsigned int out_height() const = 0; - /** Instantiate and configure a prepareB Kernel - * - * @param[in] b Input tensor B. - * @param[in] transformed_b Reshaped tensor B. - * @param[in] params GM, N, K sizes. - * @param[in] ci CPUInfo to be used for kernel configuration. - * - * @return A wrapped specialized prepareB kernel - */ - virtual std::unique_ptr instantiate_prepareB(const ITensor *b, - ITensor *transformed_b, - const INEGEMMWrapperKernel::Params ¶ms, - const CPUInfo &ci) = 0; - /** Instantiate and configure a transformA Kernel - * - * @param[in] a Input tensor A. - * @param[in] transformed_a Reshaped tensor A. - * @param[in] block_walker Window representing the layout of the matrix's blocks. - * @param[in] params M, N, K sizes. - * @param[in] gemm_info GEMM meta-data - * - * @return A wrapped specialized transformA kernel - */ - virtual std::unique_ptr instantiate_transformA(const ITensor *a, - ITensor *transformed_a, - const Window &block_walker, - const INEGEMMWrapperKernel::Params ¶ms, - const GEMMInfo &gemm_info) = 0; - /** Instantiate and configure a prepareB Kernel - * - * @param[in] transformed_a Already reshaped tensor A. - * @param[in] transformed_b Already reshaped tensor B. - * @param[in] tmp_c Temporary buffer to be used to store intermediate results. - * @param[in] c Result tensor C. - * @param[in] block_walker Window containing iteration information for the M and batch dimensions. - * @param[in] block_sizes Block sizes to use for the matrix multiplication (A & B must have been reshaped using these same block sizes). - * @param[in] params M, N, K sizes. - * @param[in] alpha Alpha value - * @param[in] beta Beta value - * @param[in] gemm_info GEMM meta-data - * @param[in] num_threads Maximum number of threads that might be used for the calculations. - * - * @return A wrapped specialized MatrixMultiply kernel - */ - virtual std::unique_ptr instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, - const Window &block_walker, const BlockSizes &block_sizes, - const INEGEMMWrapperKernel::Params ¶ms, float alpha, float beta, const GEMMInfo &gemm_info, - unsigned int num_threads) = 0; - /** Calculates the block sizes of a given strategy - * - * @param[in] ci CPUInfo to be used for kernel configuration. - * @param[in] params M, N, K sizes. - * - * @return BlockSizes for a given strategy - */ - virtual BlockSizes calculate_block_sizes_for_strategy(const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) = 0; -}; - -/** Interleaved Strategy class */ -template -class InterleavedStrategy : public IInterleavedStrategy -{ -public: - using strategy = StrategyType; - -public: - // Inherited methods overridden - unsigned int out_height() const override - { - return strategy::out_height(); - } - std::unique_ptr instantiate_prepareB(const ITensor *b, - ITensor *transformed_b, - const INEGEMMWrapperKernel::Params ¶ms, - const CPUInfo &ci) override - { - auto prepare_b = support::cpp14::make_unique>(); - prepare_b->configure(b, transformed_b, false, ci, params); - return std::move(prepare_b); - } - std::unique_ptr instantiate_transformA(const ITensor *a, - ITensor *transformed_a, - const Window &block_walker, - const INEGEMMWrapperKernel::Params ¶ms, - const GEMMInfo &gemm_info) override - { - auto transform_a = support::cpp14::make_unique>(); - transform_a->configure(a, transformed_a, false, gemm_info.reinterpret_input_as_3d(), block_walker, params); - return std::move(transform_a); - } - std::unique_ptr instantiate_matrix_multiply(const ITensor *transformed_a, const ITensor *transformed_b, ITensor *tmp_c, ITensor *c, - const Window &block_walker, const BlockSizes &block_sizes, - const INEGEMMWrapperKernel::Params ¶ms, float alpha, float beta, const GEMMInfo &gemm_info, - unsigned int num_threads) override - { - auto matrix_multiply = support::cpp14::make_unique>(); - matrix_multiply->configure(transformed_a, transformed_b, tmp_c, c, block_walker, block_sizes, params, gemm_info, alpha, beta, num_threads); - return std::move(matrix_multiply); - } - - BlockSizes calculate_block_sizes_for_strategy(const CPUInfo &ci, const INEGEMMWrapperKernel::Params ¶ms) override - { - return calculate_block_sizes(ci, params.M, params.N, params.K); - } -}; - -/** Create the backend GEMM strategy to use given the provided kernel info - * - * @param[in] kernel_name Kernel name of the backend strategy to instantiate - * - * @return The requested kernel strategy if exists else nullptr - */ -std::unique_ptr create_strategy(const std::string &kernel_name) -{ -#if defined(__arm__) - if(kernel_name.find("sgemm_8x6") != std::string::npos) - { - return support::cpp14::make_unique>(); - } -#endif // defined(__arm__) -#if defined(__aarch64__) - if(kernel_name.find("gemm_s8_4x4") != std::string::npos) - { - return support::cpp14::make_unique>(); - } - if(kernel_name.find("gemm_s8_12x8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } - if(kernel_name.find("gemm_u8_4x4") != std::string::npos) - { - return support::cpp14::make_unique>(); - } - if(kernel_name.find("gemm_u8_12x8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) - if(kernel_name.find("hgemm_24x8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } -#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) - if(kernel_name.find("sgemm_12x8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } -#if defined(__ARM_FEATURE_SVE) - if(kernel_name.find("interleaved_fp16_mla_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } - if(kernel_name.find("interleaved_fp32_mla_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } - if(kernel_name.find("interleaved_s8s32_dot_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } - if(kernel_name.find("interleaved_u8u32_dot_3VLx8") != std::string::npos) - { - return support::cpp14::make_unique>(); - } -#endif // defined(__ARM_FEATURE_SVE) -#endif // defined(__aarch64__)_ - return nullptr; -} -} // namespace detail -} // namespace arm_compute -#endif /* __ARM_COMPUTE_NEGEMMINTERLEAVEDSTRATEGIES_H__ */ diff --git a/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp b/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp deleted file mode 100644 index ecdb5a938c..0000000000 --- a/src/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h" - -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/core/WindowIterator.h" - -#include "../arm_gemm/utils.hpp" -#include "arm_gemm.hpp" - -#include "../arm_gemm/mergeresults.hpp" -#include "../arm_gemm/transform.hpp" - -#include "../arm_gemm/kernels/a64_sgemm_native_16x4.hpp" - -namespace arm_compute -{ -namespace -{ -template -struct Kernel -{ -}; - -#ifdef __aarch64__ -template <> -struct Kernel -{ - using strategy = arm_gemm::sgemm_native_16x4; -}; -#endif /* __aarch64__ */ - -} // namespace - -template -Window NEGEMMNativeWrapperKernel::configure_internal(float alpha, float beta) -{ - ARM_COMPUTE_UNUSED(alpha); - using strategy = typename Kernel::strategy; - - _beta = beta; - - //Note: The window is shifted down by 1 dimension compare to the tensors - Window window; - window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_params.M, strategy::out_height()), strategy::out_height())); - window.set(Window::DimY, Window::Dimension(0, _params.batches)); - window.set(Window::DimZ, Window::Dimension(0, _params.multis)); - - return window; -} - -template -void NEGEMMNativeWrapperKernel::run_internal(const Window &window, const Coordinates &start_offset, const Coordinates &end_offset, const ThreadInfo &info) -{ - using strategy = typename Kernel::strategy; - - TensorAccessor a(*_a); - TensorAccessor b(*_b); - TensorAccessor c(*_c); - - // Handle 3d input re-interpretation - if(_gemm_info.reinterpret_input_as_3d()) - { - Strides a_strides_as_3d = _a->info()->strides_in_bytes(); - a_strides_as_3d.remove(Window::DimZ); - a.set_strides(a_strides_as_3d); - } - - // Handle 3d output re-interpretation - if(_gemm_info.depth_output_gemm3d() != 0) - { - Strides c_strides_as_3d = _c->info()->strides_in_bytes(); - c_strides_as_3d.remove(Window::DimZ); - c.set_strides(c_strides_as_3d); - } - - unsigned int m_end = 0; - - strategy strat(info.cpu_info); - auto window_iterator = arm_compute::create_window_iterator(window, start_offset, end_offset, [&](const Coordinates & id) - { - const unsigned int y0 = id.x(); - const unsigned int batch = id.y(); - const unsigned int multi = id.z(); - const unsigned int ymax = std::min(y0 + strategy::out_height(), m_end); - - // TODO(COMPMID-1424) : Agree on gemm IO layouts - strat.kernel(a(0, y0, batch, multi), a.stride(Window::DimY), - b(0, 0, multi), b.stride(Window::DimY), - c(0, y0, batch, multi), c.stride(Window::DimY), - _beta, (ymax - y0), _params.N, _params.K); - }); - - auto on_new_row_size = [&](unsigned int start, unsigned int end) - { - ARM_COMPUTE_UNUSED(start); - m_end = std::min(end, _params.M); - }; - - window_iterator.iterate_3D(on_new_row_size); -} - -#ifdef __aarch64__ -template class NEGEMMNativeWrapperKernel; -#endif /* __aarch64__ */ - -} // namespace arm_compute diff --git a/src/runtime/NEON/functions/NEGEMM.cpp b/src/runtime/NEON/functions/NEGEMM.cpp index df92b7999c..baa22b7d32 100644 --- a/src/runtime/NEON/functions/NEGEMM.cpp +++ b/src/runtime/NEON/functions/NEGEMM.cpp @@ -34,7 +34,6 @@ #include "arm_compute/runtime/NEON/NEScheduler.h" #include "arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h" #include "arm_compute/runtime/TensorAllocator.h" -#include "support/ToolchainSupport.h" #include @@ -43,8 +42,9 @@ using namespace arm_compute::misc::shape_calculator; namespace arm_compute { NEGEMM::NEGEMM(std::shared_ptr memory_manager, IWeightsManager *weights_manager) - : _memory_group(memory_manager), _weights_manager(weights_manager), _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _asm_glue(memory_manager, weights_manager), _ma_kernel(), _tmp_a(), - _tmp_b(), _original_b(nullptr), _run_vector_matrix_multiplication(false), _run_addition(false), _reshape_b_only_on_first_run(false), _is_prepared(false) + : _memory_group(memory_manager), _weights_manager(weights_manager), _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _asm_glue(memory_manager, weights_manager), _ma_kernel(), + _alpha_scale_func(nullptr), _add_bias_kernel(), _activation_func(), _tmp_a(), _tmp_b(), _tmp_d(), _original_b(nullptr), _run_vector_matrix_multiplication(false), _run_alpha_scale(false), + _run_addition(false), _run_bias_addition(false), _run_activation(false), _reshape_b_only_on_first_run(false), _is_prepared(false) { } @@ -52,34 +52,55 @@ void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITe { ARM_COMPUTE_ERROR_THROW_ON(NEGEMM::validate(a->info(), b->info(), (c != nullptr) ? c->info() : nullptr, d->info(), alpha, beta, gemm_info)); + const bool is_c_bias = gemm_info.reshape_b_only_on_first_run(); + bool run_optimised = bool(NEGEMMAssemblyDispatch::validate(a->info(), b->info(), (is_c_bias && c != nullptr) ? c->info() : nullptr, d->info(), gemm_info)); + // Check if we need to reshape the matrix B only on the first run _is_prepared = false; _reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run(); _run_vector_matrix_multiplication = a->info()->dimension(1) < 2; _original_b = b; - - bool run_optimised = c == nullptr && bool(NEGEMMAssemblyDispatch::validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, d->info(), alpha, beta, gemm_info)); + _run_alpha_scale = alpha != 1.f; + _run_bias_addition = c != nullptr && gemm_info.reshape_b_only_on_first_run(); + _run_addition = beta != 0 && c != nullptr && !gemm_info.reshape_b_only_on_first_run(); + _run_activation = gemm_info.activation_info().enabled() && (!run_optimised || (run_optimised && !NEGEMMAssemblyDispatch::is_activation_supported(gemm_info.activation_info()))); if(run_optimised) { + const ITensor *c_to_use = is_c_bias ? c : nullptr; if(MEMInfo::get_policy() == MemoryPolicy::MINIMIZE) { GEMMInfo gemm_info_ntb = gemm_info; gemm_info_ntb.set_pretranpose_B(false); - _asm_glue.configure(a, b, c, d, alpha, beta, gemm_info_ntb); + _asm_glue.configure(a, b, c_to_use, d, gemm_info_ntb); } else { - _asm_glue.configure(a, b, c, d, alpha, beta, gemm_info); + _asm_glue.configure(a, b, c_to_use, d, gemm_info); } ARM_COMPUTE_ERROR_ON(!_asm_glue.is_configured()); + + // Scale product by alpha + if(_run_alpha_scale) + { + _alpha_scale_func.configure(d, nullptr, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LINEAR, alpha, 0.f)); + } } else { + // Pick output tensor in case bias addition should be performed + ITensor *gemm_output_to_use = d; + if(_run_bias_addition) + { + gemm_output_to_use = &_tmp_d; + _memory_group.manage(&_tmp_d); + } + + // Select between GEMV and GEMM if(_run_vector_matrix_multiplication) { // Configure the matrix multiply kernel - _mm_kernel.configure(a, b, d, alpha, false); + _mm_kernel.configure(a, b, gemm_output_to_use, alpha, false); } else { @@ -117,7 +138,7 @@ void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITe _transpose_kernel.configure(b, &_tmp_b); // Configure matrix multiplication kernel - _mm_kernel.configure(&_tmp_a, &_tmp_b, d, alpha, true, GEMMReshapeInfo(m, n, k)); + _mm_kernel.configure(&_tmp_a, &_tmp_b, gemm_output_to_use, alpha, true, GEMMReshapeInfo(m, n, k)); // Allocate once the all configure methods have been called _tmp_a.allocator()->allocate(); @@ -127,18 +148,31 @@ void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITe } } - // Configure matrix addition kernel - if(beta != 0 && c != nullptr) + if(_run_bias_addition) { - _ma_kernel.configure(c, d, beta); - _run_addition = true; + _add_bias_kernel.configure(gemm_output_to_use, c, d, ConvertPolicy::SATURATE); + _tmp_d.allocator()->allocate(); } } + + // Configure matrix addition kernel + if(_run_addition) + { + _ma_kernel.configure(c, d, beta); + } + + // Configure activation + const ActivationLayerInfo &activation = gemm_info.activation_info(); + if(_run_activation) + { + _activation_func.configure(d, nullptr, activation); + } } Status NEGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, float alpha, float beta, const GEMMInfo &gemm_info) { ARM_COMPUTE_UNUSED(alpha); + const bool is_c_bias = gemm_info.reshape_b_only_on_first_run(); ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(a); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::F16, DataType::F32); @@ -147,7 +181,7 @@ Status NEGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITenso ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported"); - if(c != nullptr) + if(c != nullptr && !is_c_bias) { ARM_COMPUTE_RETURN_ERROR_ON(gemm_info.depth_output_gemm3d() != 0); ARM_COMPUTE_RETURN_ERROR_ON(gemm_info.reinterpret_input_as_3d()); @@ -178,7 +212,7 @@ Status NEGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITenso } // Check if we need to run the optimized assembly kernel - const bool run_optimised = c == nullptr && bool(NEGEMMAssemblyDispatch::validate(a, b, c, output, alpha, beta, gemm_info)); + const bool run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, is_c_bias ? c : nullptr, output, gemm_info)); if(!run_optimised) { @@ -225,14 +259,26 @@ Status NEGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITenso // Validate matrix multiply auto_init_if_empty(tmp_output_info, matrix_a_info->clone()->set_tensor_shape(compute_mm_shape(*matrix_a_info, *matrix_b_info, run_interleave_transpose, reshape_info))); ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, &tmp_output_info, alpha, run_interleave_transpose, reshape_info)); + + if(c != nullptr && gemm_info.reshape_b_only_on_first_run()) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEArithmeticAdditionKernel::validate(&tmp_output_info, c, output, ConvertPolicy::SATURATE)); + } } // Validate matrix addition kernel - if(beta != 0 && c != nullptr) + if(beta != 0 && c != nullptr && !is_c_bias) { ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixAdditionKernel::validate(c, output, beta)); } + // Validate activation + const ActivationLayerInfo &activation = gemm_info.activation_info(); + if(activation.enabled()) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEActivationLayer::validate(output, nullptr, activation)); + } + return Status{}; } @@ -245,6 +291,10 @@ void NEGEMM::run() if(_asm_glue.is_configured()) { _asm_glue.run(); + if(_run_alpha_scale) + { + _alpha_scale_func.run(); + } } else { @@ -262,12 +312,24 @@ void NEGEMM::run() NEScheduler::get().schedule(&_mm_kernel, _run_vector_matrix_multiplication ? Window::DimX : Window::DimY); - // Run matrix addition kernel - if(_run_addition) + // Run bias addition kernel + if(_run_bias_addition) { - NEScheduler::get().schedule(&_ma_kernel, Window::DimY); + NEScheduler::get().schedule(&_add_bias_kernel, Window::DimY); } } + + // Run matrix addition kernel + if(_run_addition) + { + NEScheduler::get().schedule(&_ma_kernel, Window::DimY); + } + + // Run activation function + if(_run_activation) + { + _activation_func.run(); + } } void NEGEMM::prepare() diff --git a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp index 956ded55d2..b31ecb91e9 100644 --- a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp +++ b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp @@ -24,10 +24,8 @@ #include "arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h" #include "arm_compute/core/CPP/Validate.h" -#include "arm_compute/core/NEON/kernels/assembly/NEGEMMNativeWrapperKernel.h" #include "arm_compute/runtime/NEON/NEScheduler.h" #include "arm_compute/runtime/NEON/functions/NESimpleAssemblyFunction.h" -#include "arm_compute/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.h" #include @@ -35,43 +33,36 @@ namespace arm_compute { namespace { -std::unique_ptr create_function_all_types(const arm_gemm::KernelDescription &gemm_kernel_info, - const ITensor *a, const ITensor *b, ITensor *d, - float alpha, float beta, const GEMMInfo &gemm_info, - std::shared_ptr memory_manager, - IWeightsManager *weights_manager) - +arm_gemm::Activation map_to_arm_gemm_activation(const ActivationLayerInfo &act) { - // Note: It's safe to not check for FP16 support because this was already checked in NEGEMMAssemblyDispatch::configure() - switch(gemm_kernel_info.method) + arm_gemm::Activation gemm_act; + + // Early exit in case lower bound is other than 0, as it's not yet supported + if(act.b() != 0.f) { - case arm_gemm::GemmMethod::GEMM_INTERLEAVED: - { - if(!gemm_info.pretranpose_B()) - { - return nullptr; - } - auto function = support::cpp14::make_unique(memory_manager, weights_manager); - function->configure(a, b, d, alpha, beta, gemm_info); - return std::move(function); - } -#if defined(__aarch64__) - case arm_gemm::GemmMethod::GEMM_NATIVE: - { - if(gemm_kernel_info.name.find("sgemm_native_16x4") != std::string::npos) - { - auto kernel = support::cpp14::make_unique>(); - kernel->configure(a, b, d, alpha, beta, gemm_info); - auto function = support::cpp14::make_unique(); - function->configure(std::move(kernel)); - return std::move(function); - } - return nullptr; - } -#endif // defined(__aarch64__) + return gemm_act; + } + + switch(act.activation()) + { + case ActivationLayerInfo::ActivationFunction::RELU: + gemm_act.type = arm_gemm::Activation::Type::ReLU; + break; + case ActivationLayerInfo::ActivationFunction::BOUNDED_RELU: + gemm_act.type = arm_gemm::Activation::Type::BoundedReLU; + gemm_act.param1 = act.a(); + gemm_act.param2 = 0.f; + break; + case ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU: + gemm_act.type = arm_gemm::Activation::Type::BoundedReLU; + gemm_act.param1 = act.a(); + gemm_act.param2 = act.b(); + break; default: - return nullptr; + gemm_act.type = arm_gemm::Activation::Type::None; } + + return gemm_act; } template @@ -161,7 +152,7 @@ public: * @param[in] os Output stage meta-data. */ void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, - arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, + arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, MemoryGroup &memory_group, IWeightsManager *weights_manager, const OutputStage &os = {}); // Inherited methods overridden: @@ -214,7 +205,7 @@ private: template void Fallback::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, - arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, + arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, MemoryGroup &memory_group, IWeightsManager *weights_manager, const OutputStage &os) { arm_gemm::GemmConfig gemm_cfg; @@ -287,7 +278,7 @@ void Fallback::prepare() // Setup up matrix bias in the assembly kernel, it's just a pointer to matrix C. if(_c && _c->info()->data_type() == DataType::S32) { - _gemm_kernel_asm->set_quantized_bias(reinterpret_cast(_c->buffer() + _c->info()->offset_first_element_in_bytes())); + _gemm_kernel_asm->set_quantized_bias(reinterpret_cast(_c->buffer() + _c->info()->offset_first_element_in_bytes()), 0); } // Pretranspose B if required @@ -383,83 +374,76 @@ void Fallback::run() // Prepare assembly kernel prepare(); + TypeOutput *bias = nullptr; + // Setup up matrix bias in the assembly kernel, it's just a pointer to matrix C. + if(_c && _c->info()->data_type() != DataType::S32) + { + bias = reinterpret_cast(_c->buffer() + _c->info()->offset_first_element_in_bytes()); + } // Set gemm parameters - _gemm_kernel_asm->set_arrays(in0_ptr, lda, batch_stride_a, multi_stride_a, in1_ptr, ldb, multi_stride_b, out_ptr, ldd, batch_stride_d, multi_stride_d); + _gemm_kernel_asm->set_arrays(in0_ptr, lda, batch_stride_a, multi_stride_a, + in1_ptr, ldb, multi_stride_b, + out_ptr, ldd, batch_stride_d, multi_stride_d, + bias, 0); // Schedule assembly kernel NEScheduler::get().schedule(_optimised_kernel.get(), Window::DimX); } template -void create_function_or_arm_gemm(std::unique_ptr &acl_function, std::unique_ptr &arm_gemm, MemoryGroup &memory_group, - const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info, - std::shared_ptr memory_manager, IWeightsManager *weights_manager) +void create_arm_gemm(std::unique_ptr &arm_gemm, MemoryGroup &memory_group, + const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, arm_gemm::Activation activation, const GEMMInfo &gemm_info, + IWeightsManager *weights_manager) { INEGEMMWrapperKernel::Params p = INEGEMMWrapperKernel::extract_parameters(a, b, d, gemm_info); const CPUInfo &ci = NEScheduler::get().cpu_info(); unsigned int num_threads = NEScheduler::get().num_threads(); - arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, gemm_info.pretranpose_B()); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, gemm_info.pretranpose_B()); - // Try to create an ACL function: - const arm_gemm::KernelDescription gemm_kernel_info = arm_gemm::get_gemm_method(args); - acl_function = create_function_all_types(gemm_kernel_info, a, b, d, alpha, beta, gemm_info, std::move(memory_manager), weights_manager); - - // If we still don't have an ACL function: - if(acl_function == nullptr) - { - //Fallback onto arm_gemm function if ACL doesn't support this method. - auto fallback = support::cpp14::make_unique>(); - fallback->configure(a, b, c, d, args, gemm_info, memory_group, weights_manager); - arm_gemm = std::move(fallback); - } + // Create arm_gemm fallback + auto fallback = support::cpp14::make_unique>(); + fallback->configure(a, b, c, d, args, gemm_info, memory_group, weights_manager); + arm_gemm = std::move(fallback); } template -void create_function_or_arm_gemm_quant(std::unique_ptr &acl_function, std::unique_ptr &arm_gemm, MemoryGroup &memory_group, - const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info, - std::shared_ptr memory_manager, IWeightsManager *weights_manager) +void create_arm_gemm_quant(std::unique_ptr &arm_gemm, MemoryGroup &memory_group, + const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, arm_gemm::Activation activation, const GEMMInfo &gemm_info, + IWeightsManager *weights_manager) { INEGEMMWrapperKernel::Params p = INEGEMMWrapperKernel::extract_parameters(a, b, d, gemm_info); const CPUInfo &ci = NEScheduler::get().cpu_info(); unsigned int num_threads = NEScheduler::get().num_threads(); - arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, gemm_info.pretranpose_B()); + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, gemm_info.pretranpose_B()); // Configure requantization info const int32_t a_offset = -a->info()->quantization_info().uniform().offset; const int32_t b_offset = -b->info()->quantization_info().uniform().offset; const GEMMLowpOutputStageInfo os_info = gemm_info.gemmlowp_output_stage(); - const arm_gemm::ARequantizeLayer32 gemm_requant_info(nullptr, + const arm_gemm::ARequantizeLayer32 gemm_requant_info(nullptr, 0, a_offset, b_offset, os_info.gemmlowp_offset, -os_info.gemmlowp_shift, os_info.gemmlowp_multiplier, os_info.gemmlowp_min_bound, os_info.gemmlowp_max_bound); - // Try to create an ACL function: - const arm_gemm::KernelDescription gemm_kernel_info = arm_gemm::get_gemm_method(args, gemm_requant_info); - acl_function = create_function_all_types(gemm_kernel_info, a, b, d, alpha, beta, gemm_info, std::move(memory_manager), weights_manager); - - // If we still don't have an ACL function: - if(acl_function == nullptr) - { - // Fallback onto arm_gemm function if ACL doesn't support this method. - auto fallback = support::cpp14::make_unique>(); - fallback->configure(a, b, c, d, args, gemm_info, memory_group, weights_manager, gemm_requant_info); - arm_gemm = std::move(fallback); - } + // Create arm_gemm fallback + auto fallback = support::cpp14::make_unique>(); + fallback->configure(a, b, c, d, args, gemm_info, memory_group, weights_manager, gemm_requant_info); + arm_gemm = std::move(fallback); } } //namespace NEGEMMAssemblyDispatch::NEGEMMAssemblyDispatch(std::shared_ptr memory_manager, IWeightsManager *weights_manager) - : _function(nullptr), _arm_gemm(nullptr), _memory_group(memory_manager), _memory_manager(memory_manager), _weights_manager(weights_manager) + : _arm_gemm(nullptr), _memory_group(std::move(memory_manager)), _weights_manager(weights_manager) { } -Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, float alpha, float beta, const GEMMInfo &gemm_info) +Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, const GEMMInfo &gemm_info) { - ARM_COMPUTE_UNUSED(alpha, beta, gemm_info); + ARM_COMPUTE_UNUSED(gemm_info); ARM_COMPUTE_UNUSED(c); ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(a, b, d); ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(a); @@ -476,12 +460,19 @@ Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo return Status{}; } -void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info) +bool NEGEMMAssemblyDispatch::is_activation_supported(const ActivationLayerInfo &activation) +{ + arm_gemm::Activation act = map_to_arm_gemm_activation(activation); + return act.type != arm_gemm::Activation::Type::None; +} + +void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, const GEMMInfo &gemm_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, d); + arm_gemm::Activation act = map_to_arm_gemm_activation(gemm_info.activation_info()); //If we don't support a combination of data types, silently return: it is the caller's responsibility to check if configure() was successful via is_configured() - if(!NEGEMMAssemblyDispatch::validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, d->info(), alpha, beta, gemm_info)) + if(!NEGEMMAssemblyDispatch::validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, d->info(), gemm_info)) { return; } @@ -489,27 +480,27 @@ void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, const switch(a->info()->data_type()) { case DataType::F32: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager, _weights_manager); + create_arm_gemm(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); break; #ifdef __aarch64__ case DataType::U8: case DataType::QASYMM8: if(d->info()->data_type() == DataType::S32) { - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager, _weights_manager); + create_arm_gemm(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); } else { - create_function_or_arm_gemm_quant(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager, _weights_manager); + create_arm_gemm_quant(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); } break; case DataType::S8: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager, _weights_manager); + create_arm_gemm(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); break; #endif /* __aarch64__ */ #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC case DataType::F16: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager, _weights_manager); + create_arm_gemm(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); break; #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ default: @@ -519,33 +510,20 @@ void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, const void NEGEMMAssemblyDispatch::prepare() { - if(_function != nullptr) - { - _function->prepare(); - } - else - { - ARM_COMPUTE_ERROR_ON(_arm_gemm == nullptr); - _arm_gemm->prepare(); - } + ARM_COMPUTE_ERROR_ON(_arm_gemm == nullptr); + _arm_gemm->prepare(); } bool NEGEMMAssemblyDispatch::is_configured() const { - return (_arm_gemm != nullptr && _arm_gemm->is_configured()) || _function != nullptr; + return _arm_gemm != nullptr && _arm_gemm->is_configured(); } void NEGEMMAssemblyDispatch::run() { MemoryGroupResourceScope scope_mg(_memory_group); - if(_function != nullptr) - { - _function->run(); - } - else - { - ARM_COMPUTE_ERROR_ON(_arm_gemm == nullptr); - _arm_gemm->run(); - } + + ARM_COMPUTE_ERROR_ON(_arm_gemm == nullptr); + _arm_gemm->run(); } } //namespace arm_compute diff --git a/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp b/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp index 0034dd2545..f4377cdaf2 100644 --- a/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp +++ b/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp @@ -29,9 +29,7 @@ #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "arm_compute/runtime/NEON/NEScheduler.h" -#include "support/ToolchainSupport.h" -#include #include #include @@ -90,19 +88,27 @@ void NEConvolutionLayerReshapeWeights::run() NEGEMMConvolutionLayer::NEGEMMConvolutionLayer(const std::shared_ptr &memory_manager, IWeightsManager *weights_manager) : _memory_group(memory_manager), _weights_manager(weights_manager), _reshape_weights(), _reshape_weights_managed(), _im2col_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), - _col2im_kernel(), _activationlayer_function(), _add_bias_kernel(), _reshape_layer(), _original_weights(nullptr), _im2col_output(), _weights_reshaped(), _gemm_output(), _tmp_output(), - _data_layout(DataLayout::NCHW), _append_bias(false), _skip_im2col(false), _skip_col2im(false), _is_quantized(false), _is_activationlayer_enabled(false), _is_prepared(false) + _col2im_kernel(), _reshape_layer(), _original_weights(nullptr), _im2col_output(), _weights_reshaped(), _gemm_output(), _tmp_output(), _data_layout(DataLayout::NCHW), _skip_im2col(false), + _skip_col2im(false), _is_quantized(false), _is_prepared(false) { } void NEGEMMConvolutionLayer::configure_mm(const ITensor *input, const ITensor *weights, const ITensor *biases, ITensor *output, const ActivationLayerInfo &act_info, int gemm_3d_depth) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights); - ARM_COMPUTE_ERROR_THROW_ON(validate_mm(input->info(), weights->info(), biases == nullptr ? nullptr : biases->info(), output == nullptr ? nullptr : output->info(), act_info, gemm_3d_depth, - _skip_im2col)); + ARM_COMPUTE_ERROR_THROW_ON(validate_mm(input->info(), weights->info(), biases == nullptr ? nullptr : biases->info(), output == nullptr ? nullptr : output->info(), + act_info, gemm_3d_depth, _skip_im2col)); + // Create GEMMInfo structure const GEMMInfo &gemm_info = GEMMInfo(false, false, true /* Reshape weights only for the first run */, - gemm_3d_depth, _skip_im2col /* Reinterpret the input as 3D if im2col is skipped */); + gemm_3d_depth, _skip_im2col /* Reinterpret the input as 3D if im2col is skipped */, + false, GEMMLowpOutputStageInfo(), false, false, act_info); + + // Supported activations in GEMM + const std::set supported_acts = { ActivationLayerInfo::ActivationFunction::RELU, + ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, + ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU + }; if(_is_quantized) { @@ -125,19 +131,13 @@ void NEGEMMConvolutionLayer::configure_mm(const ITensor *input, const ITensor *w int min_activation = 0; int max_activation = 255; - const std::set supported_acts = { ActivationLayerInfo::ActivationFunction::RELU, - ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, - ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU - }; - if(_is_activationlayer_enabled && supported_acts.count(act_info.activation()) != 0) + if(supported_acts.count(act_info.activation()) != 0) { const int a_const_int = quantize_qasymm8(act_info.a(), oqinfo); const int b_const_int = quantize_qasymm8(act_info.b(), oqinfo); min_activation = act_info.activation() != ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU ? oqinfo.offset : b_const_int; max_activation = act_info.activation() == ActivationLayerInfo::ActivationFunction::RELU ? 255 : a_const_int; - - _is_activationlayer_enabled = false; } GEMMLowpOutputStageInfo output_info; @@ -157,18 +157,21 @@ void NEGEMMConvolutionLayer::configure_mm(const ITensor *input, const ITensor *w else { // Configure matrix multiply function - _mm_gemm.configure(input, weights, nullptr, output, 1.0f, 0.0f, gemm_info); + _mm_gemm.configure(input, weights, biases, output, 1.0f, 0.0f, gemm_info); } } -Status NEGEMMConvolutionLayer::validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const ActivationLayerInfo &act_info, - int gemm_3d_depth, bool skip_im2col) +Status NEGEMMConvolutionLayer::validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, + const ActivationLayerInfo &act_info, int gemm_3d_depth, bool skip_im2col) { const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type()); const bool is_activation_enabled = act_info.enabled(); - const GEMMInfo &gemm_info = GEMMInfo(false, false, true /* Reshape weights only for the first run */, - gemm_3d_depth, skip_im2col /* Reinterpret the input as 3D if im2col is skipped */); + // Create GEMMInfo structure + const GEMMInfo gemm_info = GEMMInfo(false, false, true /* Reshape weights only for the first run */, + gemm_3d_depth, skip_im2col /* Reinterpret the input as 3D if im2col is skipped */, + false, GEMMLowpOutputStageInfo(), false, false, act_info); + if(is_quantized) { // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() @@ -241,7 +244,7 @@ void NEGEMMConvolutionLayer::configure(const ITensor *input, const ITensor *weig const Size2D &dilation, const ActivationLayerInfo &act_info, unsigned int num_groups) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output); - ARM_COMPUTE_UNUSED(num_groups); + ARM_COMPUTE_UNUSED(num_groups, weights_info); ARM_COMPUTE_ERROR_THROW_ON(NEGEMMConvolutionLayer::validate(input->info(), weights->info(), biases != nullptr ? biases->info() : nullptr, @@ -261,13 +264,11 @@ void NEGEMMConvolutionLayer::configure(const ITensor *input, const ITensor *weig const unsigned int kernel_width = weights->info()->dimension(idx_width); const unsigned int kernel_height = weights->info()->dimension(idx_height); - _is_prepared = weights_info.retain_internal_weights(); - _original_weights = weights; - _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type()); - _data_layout = data_layout; - _skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1); - _append_bias = (biases != nullptr) && (!_is_quantized); - _is_activationlayer_enabled = act_info.enabled(); + _is_prepared = weights_info.retain_internal_weights(); + _original_weights = weights; + _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type()); + _data_layout = data_layout; + _skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1); const ITensor *gemm_input_to_use = input; ITensor *gemm_output_to_use = output; @@ -297,8 +298,6 @@ void NEGEMMConvolutionLayer::configure(const ITensor *input, const ITensor *weig _skip_col2im = false; } - const ITensor *biases_to_use = (_append_bias && !_skip_im2col) ? biases : nullptr; - // Get parameters from conv_info unsigned int stride_x = 0; unsigned int stride_y = 0; @@ -312,12 +311,12 @@ void NEGEMMConvolutionLayer::configure(const ITensor *input, const ITensor *weig if(_weights_manager && _weights_manager->are_weights_managed(weights)) { - _reshape_weights_managed.configure(weights, biases_to_use); + _reshape_weights_managed.configure(weights, nullptr); weights_to_use = _weights_manager->acquire(weights, &_reshape_weights_managed); } else { - _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped); + _reshape_weights.configure(weights, nullptr, &_weights_reshaped); weights_to_use = &_weights_reshaped; } @@ -327,16 +326,11 @@ void NEGEMMConvolutionLayer::configure(const ITensor *input, const ITensor *weig _memory_group.manage(&_im2col_output); // Configure - _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, _append_bias, dilation); + _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, false, dilation); // Update GEMM input gemm_input_to_use = &_im2col_output; } - else if(_append_bias) - { - // Configure add bias kernel - _add_bias_kernel.configure(output, biases, output, ConvertPolicy::SATURATE); - } // Create temporary GEMM output tensor in case we cannot skip col2im if(!_skip_col2im) @@ -394,14 +388,6 @@ void NEGEMMConvolutionLayer::configure(const ITensor *input, const ITensor *weig ARM_COMPUTE_ERROR_ON_MSG((output->info()->dimension(idx_width) != conv_w) || (output->info()->dimension(idx_height) != conv_h), "Output shape does not match the expected one"); - - // Configure Activation Layer - if(_is_activationlayer_enabled) - { - _activationlayer_function.configure(output, nullptr, act_info); - } - - ARM_COMPUTE_UNUSED(weights_info); } Status NEGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info, @@ -432,10 +418,9 @@ Status NEGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorI const ITensorInfo *gemm_output_to_use = output; const ITensorInfo *weights_to_use = weights; - const bool is_quantized = is_data_type_quantized_asymmetric(data_type); - const bool append_bias = (biases != nullptr) && (!is_quantized); - bool skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1); - bool is_activation_enabled = act_info.enabled(); + const bool append_bias = false; + const bool is_quantized = is_data_type_quantized_asymmetric(data_type); + bool skip_im2col = (data_layout == DataLayout::NHWC && kernel_width == 1 && kernel_height == 1 && conv_info.stride().first == 1 && conv_info.stride().second == 1); // Get convolved dimensions unsigned int conv_w = 0; @@ -470,9 +455,6 @@ Status NEGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorI } } - const unsigned bias_element = (append_bias && !skip_im2col) ? 1 : 0; - const ITensorInfo *biases_to_use = (append_bias && !skip_im2col) ? biases : nullptr; - ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_channel) != input->dimension(idx_channel)); ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 4); @@ -491,17 +473,12 @@ Status NEGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorI ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1); } - if(act_info.enabled()) - { - ARM_COMPUTE_ERROR_ON(act_info.b() > act_info.a()); - } - unsigned int mat_weights_cols = weights->dimension(idx_kernels); - unsigned int mat_weights_rows = weights->dimension(idx_width) * weights->dimension(idx_height) * weights->dimension(idx_channel) + bias_element; + unsigned int mat_weights_rows = weights->dimension(idx_width) * weights->dimension(idx_height) * weights->dimension(idx_channel); // Output tensor auto inizialization if not yet initialized - ARM_COMPUTE_RETURN_ON_ERROR(NEConvolutionLayerReshapeWeights::validate(weights, biases_to_use, nullptr)); - weights_reshaped_info = TensorInfo(compute_weights_reshaped_shape(*weights, (append_bias && !skip_im2col)), 1, data_type); + ARM_COMPUTE_RETURN_ON_ERROR(NEConvolutionLayerReshapeWeights::validate(weights, nullptr, nullptr)); + weights_reshaped_info = TensorInfo(compute_weights_reshaped_shape(*weights, append_bias), 1, data_type); weights_reshaped_info.set_quantization_info(weights->quantization_info()); weights_to_use = &weights_reshaped_info; @@ -521,11 +498,6 @@ Status NEGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorI ARM_COMPUTE_RETURN_ON_ERROR(NEIm2ColKernel::validate(input, &im2col_reshaped_info, Size2D(kernel_width, kernel_height), conv_info, append_bias, dilation)); gemm_input_to_use = &im2col_reshaped_info; } - else if(append_bias) - { - // Validate add bias kernel - ARM_COMPUTE_RETURN_ON_ERROR(NEArithmeticAdditionKernel::validate(output, biases, output, ConvertPolicy::SATURATE)); - } // Create temporary GEMM output tensor in case we cannot skip col2im if(!skip_col2im) @@ -549,12 +521,6 @@ Status NEGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorI ARM_COMPUTE_RETURN_ON_ERROR(NECol2ImKernel::validate(gemm_output_to_use, output, Size2D(conv_w, conv_h))); } - //Validate Activation Layer - if(is_activation_enabled) - { - ARM_COMPUTE_RETURN_ON_ERROR(NEActivationLayer::validate(output, nullptr, act_info)); - } - return Status{}; } @@ -583,11 +549,6 @@ void NEGEMMConvolutionLayer::run() _mm_gemm.run(); } - if(_skip_im2col && _append_bias) - { - NEScheduler::get().schedule(&_add_bias_kernel, Window::DimY); - } - // Reshape output matrix if(!_skip_col2im) { @@ -600,11 +561,6 @@ void NEGEMMConvolutionLayer::run() _reshape_layer.run(); } } - - if(_is_activationlayer_enabled) - { - _activationlayer_function.run(); - } } void NEGEMMConvolutionLayer::prepare() diff --git a/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp b/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp index aa40113c5e..346d025fd2 100644 --- a/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp +++ b/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp @@ -59,7 +59,7 @@ void NEGEMMLowpAssemblyMatrixMultiplyCore::configure(const ITensor *a, const ITe case DataType::QASYMM8: case DataType::U8: { - _asm_glue.configure(a, b, c, output, 1.f, 0.f, GEMMInfo(false, false, true)); + _asm_glue.configure(a, b, c, output, GEMMInfo(false, false, true)); run_optimised = _asm_glue.is_configured(); break; } diff --git a/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp b/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp index a03ec108c6..617d66cf24 100644 --- a/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp +++ b/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp @@ -42,8 +42,9 @@ using namespace arm_compute::misc::shape_calculator; NEGEMMLowpMatrixMultiplyCore::NEGEMMLowpMatrixMultiplyCore(std::shared_ptr memory_manager) : _memory_group(memory_manager), _asm_glue(memory_manager), _mm_kernel(nullptr), _mtx_a_reshape_kernel(nullptr), _mtx_b_reshape_kernel(nullptr), _mtx_a_reduction_kernel(), _mtx_b_reduction_kernel(), - _offset_contribution_kernel(), _offset_contribution_output_stage_kernel(), _vector_sum_col(), _vector_sum_row(), _tmp_a(), _tmp_b(), _mm_result_s32(), _original_b(nullptr), _a_offset(0), _b_offset(0), - _run_vector_matrix_multiplication(false), _assembly_path(false), _fused_assembly_path(false), _reshape_b_only_on_first_run(false), _is_prepared(false), _fuse_output_stage(false) + _offset_contribution_kernel(), _offset_contribution_output_stage_kernel(), _activation_func(), _vector_sum_col(), _vector_sum_row(), _tmp_a(), _tmp_b(), _mm_result_s32(), _original_b(nullptr), + _a_offset(0), _b_offset(0), _run_vector_matrix_multiplication(false), _assembly_path(false), _fused_assembly_path(false), _reshape_b_only_on_first_run(false), _is_prepared(false), + _fuse_output_stage(false), _run_activation(false) { } @@ -87,12 +88,12 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, { if(a->info()->data_type() == DataType::QASYMM8 && gemm_info.gemmlowp_output_stage().type == GEMMLowpOutputStageType::QUANTIZE_DOWN_FIXEDPOINT) { - _asm_glue.configure(a, b, c, output, 1.f, 0.f, gemm_info); + _asm_glue.configure(a, b, c, output, gemm_info); _fused_assembly_path = _asm_glue.is_configured(); } else { - _asm_glue.configure(a, b, nullptr, _fuse_output_stage ? &_mm_result_s32 : output, 1.f, 0.f, gemm_info); + _asm_glue.configure(a, b, nullptr, _fuse_output_stage ? &_mm_result_s32 : output, gemm_info); } _assembly_path = _asm_glue.is_configured(); break; @@ -192,6 +193,14 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, } } + // Configure activation + const ActivationLayerInfo &activation = gemm_info.activation_info(); + _run_activation = activation.enabled() && (!_assembly_path || (_assembly_path && !NEGEMMAssemblyDispatch::is_activation_supported(activation))); + if(_run_activation) + { + _activation_func.configure(output, nullptr, activation); + } + // Allocate tensors if(!_assembly_path && !_run_vector_matrix_multiplication) { @@ -253,12 +262,12 @@ Status NEGEMMLowpMatrixMultiplyCore::validate(const ITensorInfo *a, const ITenso bool run_optimised_requantized = false; if(is_data_type_quantized_asymmetric(a->data_type())) { - run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, c, output, 1.f, 0.f, gemm_info)); + run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, c, output, gemm_info)); run_optimised_requantized = run_optimised; } else { - run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, nullptr, fuse_output_stage ? &mm_result_s32_info : output, 1.f, 0.f, gemm_info)); + run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, nullptr, fuse_output_stage ? &mm_result_s32_info : output, gemm_info)); } if(run_optimised) @@ -361,6 +370,14 @@ Status NEGEMMLowpMatrixMultiplyCore::validate(const ITensorInfo *a, const ITenso a_offset, b_offset)); } } + + // Validate activation + const ActivationLayerInfo &activation = gemm_info.activation_info(); + if(activation.enabled()) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEActivationLayer::validate(output, nullptr, activation)); + } + return Status{}; } @@ -415,6 +432,12 @@ void NEGEMMLowpMatrixMultiplyCore::run() NEScheduler::get().schedule(&_offset_contribution_kernel, Window::DimY); } } + + // Run fused activation + if(_run_activation) + { + _activation_func.run(); + } } void NEGEMMLowpMatrixMultiplyCore::prepare() diff --git a/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp b/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp deleted file mode 100644 index 1aeab5b9cb..0000000000 --- a/src/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.cpp +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Copyright (c) 2018-2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "arm_compute/runtime/NEON/functions/assembly/NEGEMMInterleavedWrapper.h" - -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/NEON/kernels/assembly/Helpers.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/runtime/NEON/NEScheduler.h" - -#include "src/core/NEON/kernels/assembly/NEGEMMInterleavedStrategies.h" - -#include -#include -#include - -namespace arm_compute -{ -#ifndef NO_MULTI_THREADING -class BufferManagerMultipleThreads final : public IBufferManager -{ -public: - /** Number of buffers to ping pong between */ - static constexpr unsigned int NUM_BUFFERS = 3; - - explicit BufferManagerMultipleThreads(unsigned int max_num_users) - : _buffers(), _max_num_users(max_num_users) - { - } - unsigned int num_buffers() const override - { - return NUM_BUFFERS; - } - /* - Lock the requested index if it's free and return true if it needs reshaping. - * - Return false without acquiring the lock if the buffer at the index is already reshaped / being reshaped. - * - Block if the corresponding buffer for the given index is still being used by a different index. - */ - bool lock_to_reshape_if_needed(unsigned int index) override - { - Buffer &buf = get_buffer_from_index(index); - while(true) - { - if(buf.index == index && buf.state != State::FREE) - { - //Another thread already is reshaping / has reshaped this block: nothing to do - return false; - } - else - { - std::unique_lock lock(buf.mutex); - //If the buffer is free then lock it for reshaping: - if(buf.state == State::FREE) - { - buf.index = index; - buf.state = State::BEING_RESHAPED; - return true; - } - // Check again just in case it changed while we were acquiring the lock: - if(buf.index == index) - { - //Another thread is reshaping this block already, nothing to do - return false; - } - // buf.index != index: Buffer still being used by another block, need to wait - buf.sem.wait(lock); - } - } - } - /* Mark the buffer at the given index as reshaped and release the lock acquired via lock_to_reshape_if_needed() */ - void mark_as_reshaped(unsigned int index) override - { - Buffer &buf = get_buffer_from_index(index); - { - std::lock_guard lock(buf.mutex); - buf.users = _max_num_users; - buf.state = State::IN_USE; - } - buf.sem.notify_all(); - } - - /* Block until the buffer at the given index is reshaped */ - void wait_for_reshaping(unsigned int index) override - { - Buffer &buf = get_buffer_from_index(index); - ARM_COMPUTE_ERROR_ON(buf.index != index); // Should have blocked in lock_to_reshape_if_needed() - // Check if it's already ready to use: - if(buf.state == State::IN_USE) - { - return; - } - std::unique_lock lock(buf.mutex); - //Double check it didn't change while we were acquiring the lock: - if(buf.state == State::IN_USE) - { - return; - } - buf.sem.wait(lock); - } - /* Mark the buffer at the given index as not used by this thread anymore. - * Once all the threads have called this method then the buffer is marked as free again. - */ - void mark_as_unused(unsigned int index) override - { - Buffer &buf = get_buffer_from_index(index); - ARM_COMPUTE_ERROR_ON(buf.index != index); // Should have blocked in lock_to_reshape_if_needed() - if(--buf.users == 0) - { - std::unique_lock lock(buf.mutex); - buf.state = State::FREE; - lock.unlock(); - buf.sem.notify_all(); - } - } - -private: - enum class State - { - FREE, - BEING_RESHAPED, - IN_USE - }; - struct Buffer - { - unsigned int index{}; - std::atomic_uint users{}; - State state{ State::FREE }; - std::mutex mutex{}; - std::condition_variable sem{}; - }; - std::array _buffers; - Buffer &get_buffer_from_index(unsigned int index) - { - return _buffers[index % NUM_BUFFERS]; - } - unsigned int _max_num_users; -}; -#endif /* NO_MULTI_THREADING */ - -class BufferManagerSingleThread : public IBufferManager -{ -public: - unsigned int num_buffers() const override - { - return 1; - } - bool lock_to_reshape_if_needed(unsigned int index) override - { - ARM_COMPUTE_UNUSED(index); - return true; - } - void mark_as_reshaped(unsigned int index) override - { - ARM_COMPUTE_UNUSED(index); - } - void wait_for_reshaping(unsigned int index) override - { - ARM_COMPUTE_UNUSED(index); - } - void mark_as_unused(unsigned int index) override - { - ARM_COMPUTE_UNUSED(index); - } -}; - -NEGEMMInterleavedWrapper::NEGEMMInterleavedWrapper(std::shared_ptr memory_manager, IWeightsManager *weights_manager) - : _memory_group(std::move(memory_manager)), - _weights_manager(weights_manager) -{ -} - -void NEGEMMInterleavedWrapper::run() -{ - prepare(); - - MemoryGroupResourceScope scope_mg(_memory_group); - NEScheduler::get().run_tagged_workloads(_workloads, _tag.c_str()); -} - -void NEGEMMInterleavedWrapper::prepare() -{ - ARM_COMPUTE_UNUSED(_weights_manager); - if(!_is_prepared) - { - if(_pretranspose_b) - { - _transformed_b.allocator()->allocate(); - NEScheduler::get().schedule(_prepare_b.get(), Window::DimX); - _b->mark_as_unused(); - } - else - { - _prepare_b->create_workloads(_b_workloads); - } - _transform_a->create_workloads(_a_workloads); - _matrix_multiply->create_workloads(_mm_workloads); - - //Maximum number of workloads to create: - const unsigned int num_threads = NEScheduler::get().num_threads(); - const unsigned int max_iterations = num_threads == 1 ? 1 : num_threads; - //Maximum number of iterations the parameters allow: - const unsigned int num_iterations = _batch_window.num_iterations_total(); - // Keep the smallest of the two: - const unsigned int num_windows = std::min(num_iterations, max_iterations); - const TensorShape window_shape = _batch_window.shape(); - const unsigned int num_x_blocks = _block_walker.num_iterations(Window::DimX); - - // Create a 1D window to dynamically split the batch window: - Window win_1D; - win_1D.set(0, Window::Dimension(0, num_iterations)); - - // Create one workload for each sub-window: - for(unsigned int w = 0; w < num_windows; w++) - { - Window win = win_1D.split_window(0, w, num_windows); - const Coordinates start_offset = index2coords(window_shape, win.x().start()); - const Coordinates end_offset = index2coords(window_shape, win.x().end() - 1); - - if(_pretranspose_b) - { - auto workload = [start_offset, end_offset, num_x_blocks, this](const ThreadInfo & info) - { - //For each block of rows in "M" - auto workload_mm = this->_mm_workloads.begin(); - for(auto &workload_a : this->_a_workloads) - { - // Transform one k_block from A: - this->_transform_a->transform(workload_a, info, this->_batch_window, start_offset, end_offset); - // Then perform the matrix multiplication for each x block along N: - for(unsigned int i = 0; i < num_x_blocks; i++) - { - ARM_COMPUTE_ERROR_ON(workload_mm == this->_mm_workloads.end()); - this->_matrix_multiply->transform(*workload_mm++, info, this->_batch_window, start_offset, end_offset); - } - } - }; - _workloads.emplace_back(workload); - } - else - { - auto workload = [num_threads, start_offset, end_offset, num_x_blocks, this](const ThreadInfo & info) - { - //For each block of rows in "M" - auto workload_mm = this->_mm_workloads.begin(); - unsigned int workload_b = 0; - //If there is only one thread then only reshape the B blocks as you need them: - unsigned int workload_b_next = num_threads == 1 ? this->_b_workloads.size() : 1; - - for(auto &workload_a : this->_a_workloads) - { - // Transform one k_block from A: - this->_transform_a->transform(workload_a, info, this->_batch_window, start_offset, end_offset); - // Then perform the matrix multiplication for each x block along N: - for(unsigned int i = 0; i < num_x_blocks; i++) - { - ARM_COMPUTE_ERROR_ON(workload_mm == this->_mm_workloads.end()); - if(workload_b_next < this->_b_workloads.size()) - { - //Lock on BufferManager: need to run it ? - if(this->_buffer_manager->lock_to_reshape_if_needed(workload_b_next)) - { - this->_prepare_b->transform(this->_b_workloads[workload_b_next], info); - this->_buffer_manager->mark_as_reshaped(workload_b_next); - } - workload_b_next++; - } - ARM_COMPUTE_ERROR_ON(workload_b >= this->_b_workloads.size()); - // Run if needed or wait - if(this->_buffer_manager->lock_to_reshape_if_needed(workload_b)) - { - this->_prepare_b->transform(this->_b_workloads[workload_b], info); - this->_buffer_manager->mark_as_reshaped(workload_b); - } - this->_buffer_manager->wait_for_reshaping(workload_b); - this->_matrix_multiply->transform(*workload_mm++, info, this->_batch_window, start_offset, end_offset); - this->_buffer_manager->mark_as_unused(workload_b); - workload_b++; - } - } - }; - _workloads.emplace_back(workload); - } - } - if(!_pretranspose_b && num_windows > 1 && num_windows % num_threads != 0) - { - //Make sure the number of workloads is a multiple of the number of threads to avoid dead locks: - for(unsigned int leftover = num_windows % num_threads; leftover != num_threads; leftover++) - { - auto workload = [this](const ThreadInfo & info) - { - unsigned int workload_b = 0; - //If there is only one thread then only reshape the B blocks as you need them: - unsigned int workload_b_next = 1; - - for(unsigned int iteration = 0; iteration < this->_mm_workloads.size(); iteration++) - { - if(workload_b_next < this->_b_workloads.size()) - { - //Lock on BufferManager: need to run it ? - if(this->_buffer_manager->lock_to_reshape_if_needed(workload_b_next)) - { - this->_prepare_b->transform(this->_b_workloads[workload_b_next], info); - this->_buffer_manager->mark_as_reshaped(workload_b_next); - } - workload_b_next++; - } - ARM_COMPUTE_ERROR_ON(workload_b >= this->_b_workloads.size()); - // Run if needed or wait - if(this->_buffer_manager->lock_to_reshape_if_needed(workload_b)) - { - this->_prepare_b->transform(this->_b_workloads[workload_b], info); - this->_buffer_manager->mark_as_reshaped(workload_b); - } - this->_buffer_manager->wait_for_reshaping(workload_b); - this->_buffer_manager->mark_as_unused(workload_b); - workload_b++; - } - }; - _workloads.emplace_back(workload); - } - } - - _is_prepared = true; - } -} - -void NEGEMMInterleavedWrapper::configure(const ITensor *a, const ITensor *b, ITensor *c, float alpha, float beta, const GEMMInfo &gemm_info) -{ - _params = INEGEMMWrapperKernel::extract_parameters(a, b, c, gemm_info); - _a = a; - _b = b; - _c = c; - _pretranspose_b = gemm_info.pretranpose_B(); - - const DataType input_type = a->info()->data_type(); - const CPUInfo &ci = NEScheduler::get().cpu_info(); - const unsigned int num_threads = NEScheduler::get().num_threads(); - - const arm_gemm::KernelDescription gemm_kernel_info = get_gemm_info(input_type, ci, num_threads, _params, alpha, beta, _pretranspose_b); - ARM_COMPUTE_ERROR_ON(gemm_kernel_info.method != arm_gemm::GemmMethod::GEMM_INTERLEAVED); - - // Forcing 128-byte alignment (required by 32-bit kernels) - const unsigned int alignment = 128; - _transformed_b.allocator()->init(TensorInfo{}, alignment); - _tmp_c.allocator()->init(TensorInfo{}, alignment); - _tag = "NEGEMMInterleaved_" + gemm_kernel_info.name; - - // Get strategy - std::unique_ptr strategy = detail::create_strategy(gemm_kernel_info.name); - ARM_COMPUTE_ERROR_ON(strategy == nullptr); - - if(!_pretranspose_b) - { - _block_sizes = strategy->calculate_block_sizes_for_strategy(ci, _params); - _batch_window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_block_sizes.m_round, _block_sizes.strategy_out_height), _block_sizes.strategy_out_height)); - _batch_window.set(Window::DimY, Window::Dimension(0, _params.batches)); - // If the execution is single threaded or has only one window then the buffer manager only needs 1 buffer else we will use NUM_BUFFERS buffers and ping pong between them: - const unsigned int num_iterations = _batch_window.num_iterations_total(); - if(NEScheduler::get().num_threads() == 1 || num_iterations == 1) - { - _buffer_manager = support::cpp14::make_unique(); - } - else - { -#ifdef NO_MULTI_THREADING - ARM_COMPUTE_ERROR("Can't have more than 1 buffer without multiple threads"); -#else /* NO_MULTI_THREADING */ - _buffer_manager = support::cpp14::make_unique(NEScheduler::get().num_threads()); -#endif /* NO_MULTI_THREADING */ - } - // If B is transposed at every iteration then transformed_B can be managed: - _memory_group.manage(&_transformed_b); - auto_init_if_empty(*_transformed_b.info(), _b->info()->clone()->set_tensor_shape(TensorShape(_block_sizes.x_block * _block_sizes.k_block, _buffer_manager->num_buffers()))); - } - else - { - _tag += "_preB"; - } - - _prepare_b = strategy->instantiate_prepareB(b, &_transformed_b, _params, ci); - ARM_COMPUTE_ERROR_ON(_prepare_b == nullptr); - - if(_pretranspose_b) - { - _block_sizes = _prepare_b->block_sizes(); - _batch_window.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_block_sizes.m_round, _block_sizes.strategy_out_height), _block_sizes.strategy_out_height)); - _batch_window.set(Window::DimY, Window::Dimension(0, _params.batches)); - } - - _block_walker.set(Window::DimX, Window::Dimension(0, ceil_to_multiple(_params.N, _block_sizes.x_block), _block_sizes.x_block)); - _block_walker.set(Window::DimY, Window::Dimension(0, ceil_to_multiple(_params.K, _block_sizes.k_block), _block_sizes.k_block)); - _block_walker.set(Window::DimZ, Window::Dimension(0, _params.multis)); - - _transformed_a.allocator()->init(TensorInfo(TensorShape{ _block_sizes.k_block, _block_sizes.m_round, _params.batches }, 1, input_type), alignment); - _memory_group.manage(&_transformed_a); - _memory_group.manage(&_tmp_c); - - _transform_a = strategy->instantiate_transformA(_a, &_transformed_a, _block_walker, _params, gemm_info); - _matrix_multiply = strategy->instantiate_matrix_multiply(&_transformed_a, &_transformed_b, &_tmp_c, c, _block_walker, _block_sizes, _params, alpha, beta, gemm_info, num_threads); - ARM_COMPUTE_ERROR_ON(_transform_a == nullptr); - ARM_COMPUTE_ERROR_ON(_matrix_multiply == nullptr); - - _transformed_a.allocator()->allocate(); - _tmp_c.allocator()->allocate(); - if(!_pretranspose_b) - { - _transformed_b.allocator()->allocate(); - } -} -} // namespace arm_compute -- cgit v1.2.1